Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: support file level ingestion and files listing search (#1536)

Rico Furtadoe74483c04069
diff --git a/frontend/app/api/queries/useBrowseConnectionFiles.ts b/frontend/app/api/queries/useBrowseConnectionFiles.ts
new file mode 100644
index 000000000..fa0c12d31
--- /dev/null
+++ b/frontend/app/api/queries/useBrowseConnectionFiles.ts
@@ -0,0 +1,75 @@
+import {
+ type UseQueryOptions,
+ useQuery,
+ useQueryClient,
+} from "@tanstack/react-query";
+
+export interface RemoteFile {
+ id: string;
+ name: string;
+ bucket: string;
+ key: string;
+ size: number;
+ modified_time: string;
+ is_ingested: boolean;
+}
+
+export interface BrowseConnectionFilesParams {
+ connectorType: string;
+ connectionId: string;
+ bucket?: string;
+ search?: string;
+ pageToken?: string;
+ maxFiles?: number;
+}
+
+export interface BrowseConnectionFilesResponse {
+ files: RemoteFile[];
+ next_page_token: string | null;
+ total_remote: number;
+ total_ingested: number;
+}
+
+export const useBrowseConnectionFiles = (
+ params: BrowseConnectionFilesParams,
+ options?: Omit<
+ UseQueryOptions<BrowseConnectionFilesResponse>,
+ "queryKey" | "queryFn"
+ >,
+) => {
+ const queryClient = useQueryClient();
+
+ async function fetchFiles(): Promise<BrowseConnectionFilesResponse> {
+ const searchParams = new URLSearchParams();
+
+ if (params.bucket) searchParams.set("bucket", params.bucket);
+ if (params.search) searchParams.set("search", params.search);
+ if (params.pageToken) searchParams.set("page_token", params.pageToken);
+ if (params.maxFiles) searchParams.set("max_files", String(params.maxFiles));
+
+ const url = `/api/connectors/${params.connectorType}/${params.connectionId}/browse?${searchParams.toString()}`;
+ const response = await fetch(url);
+
+ if (!response.ok) {
+ const errorData = await response
+ .json()
+ .catch(() => ({ error: "Unknown error" }));
+ throw new Error(
+ errorData.error || `Failed to browse files: ${response.status}`,
+ );
+ }
+
+ return response.json();
+ }
+
+ return useQuery(
+ {
+ queryKey: ["browseConnectionFiles", params],
+ queryFn: fetchFiles,
+ retry: false,
+ enabled: Boolean(params.connectorType && params.connectionId),
+ ...options,
+ },
+ queryClient,
+ );
+};
diff --git a/frontend/app/api/queries/useListFiles.ts b/frontend/app/api/queries/useListFiles.ts
new file mode 100644
index 000000000..b5b06f385
--- /dev/null
+++ b/frontend/app/api/queries/useListFiles.ts
@@ -0,0 +1,97 @@
+import {
+ type UseQueryOptions,
+ useQuery,
+ useQueryClient,
+} from "@tanstack/react-query";
+import type { File } from "./useGetSearchQuery";
+
+export interface ListFilesParams {
+ page?: number;
+ pageSize?: number;
+ sortBy?: string;
+ sortOrder?: "asc" | "desc";
+ connectorType?: string;
+ mimetype?: string;
+ owner?: string;
+ search?: string;
+}
+
+export interface ListFilesResponse {
+ files: File[];
+ total: number;
+ page: number;
+ page_size: number;
+}
+
+export const useListFiles = (
+ params: ListFilesParams = {},
+ options?: Omit<UseQueryOptions<ListFilesResponse>, "queryKey" | "queryFn">,
+) => {
+ const queryClient = useQueryClient();
+
+ async function fetchFiles(): Promise<ListFilesResponse> {
+ const searchParams = new URLSearchParams();
+
+ if (params.page) searchParams.set("page", String(params.page));
+ if (params.pageSize) searchParams.set("page_size", String(params.pageSize));
+ if (params.sortBy) searchParams.set("sort_by", params.sortBy);
+ if (params.sortOrder) searchParams.set("sort_order", params.sortOrder);
+ if (params.connectorType)
+ searchParams.set("connector_type", params.connectorType);
+ if (params.mimetype) searchParams.set("mimetype", params.mimetype);
+ if (params.owner) searchParams.set("owner", params.owner);
+ if (params.search) searchParams.set("search", params.search);
+
+ const url = `/api/files?${searchParams.toString()}`;
+ const response = await fetch(url);
+
+ if (!response.ok) {
+ const errorData = await response
+ .json()
+ .catch(() => ({ error: "Unknown error" }));
+ throw new Error(
+ errorData.error || `Failed to list files: ${response.status}`,
+ );
+ }
+
+ const data = await response.json();
+
+ // Map server response to File interface
+ const files: File[] = (data.files || []).map(
+ (f: Record<string, unknown>) => ({
+ filename: (f.filename as string) || "",
+ mimetype: (f.mimetype as string) || "",
+ chunkCount: (f.chunk_count as number) || 0,
+ source_url: (f.source_url as string) || "",
+ owner: (f.owner as string) || "",
+ owner_name: (f.owner_name as string) || "",
+ owner_email: (f.owner_email as string) || "",
+ size: (f.file_size as number) || 0,
+ connector_type: (f.connector_type as string) || "local",
+ embedding_model: f.embedding_model as string | undefined,
+ embedding_dimensions: f.embedding_dimensions as number | undefined,
+ allowed_users: (f.allowed_users as string[]) || [],
+ allowed_groups: (f.allowed_groups as string[]) || [],
+ status: "active" as const,
+ }),
+ );
+
+ return {
+ files,
+ total: data.total || 0,
+ page: data.page || 1,
+ page_size: data.page_size || 25,
+ };
+ }
+
+ return useQuery(
+ {
+ queryKey: ["listFiles", params],
+ placeholderData: (prev: ListFilesResponse | undefined) => prev,
+ queryFn: fetchFiles,
+ retry: false,
+ ...options,
+ },
+ queryClient,
+ );
+};
diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsx
index c7a4d0c79..6fa0a7eed 100644
--- a/frontend/app/knowledge/page.tsx
+++ b/frontend/app/knowledge/page.tsx
@@ -25,6 +25,7 @@ import {
type SearchResult,
useGetSearchQuery,
} from "../api/queries/useGetSearchQuery";
+import { useListFiles } from "../api/queries/useListFiles";
import "@/components/AgGrid/registerAgGridModules";
import "@/components/AgGrid/agGridStyles.css";
import { toast } from "sonner";
@@ -209,17 +210,52 @@ function SearchPage() {
getFailedFileKey,
]);
+ // Use server-side file listing for default/wildcard view,
+ // fall back to search-based aggregation for semantic queries
+ const isWildcardQuery =
+ !queryOverride ||
+ queryOverride.trim() === "" ||
+ queryOverride.trim() === "*";
+
+ const {
+ data: listFilesData,
+ isLoading: isListFilesLoading,
+ error: listFilesError,
+ isError: isListFilesError,
+ } = useListFiles(
+ {
+ pageSize: 100,
+ search: isWildcardQuery ? undefined : queryOverride,
+ connectorType: parsedFilterData?.filters?.connector_types?.[0],
+ mimetype: parsedFilterData?.filters?.document_types?.[0],
+ owner: parsedFilterData?.filters?.owners?.[0],
+ },
+ {
+ refetchInterval: 5000,
+ enabled: isWildcardQuery,
+ },
+ );
+
const {
data: searchData = EMPTY_SEARCH_RESULT,
isLoading: isSearchLoading,
- error,
- isError,
+ error: searchError,
+ isError: isSearchError,
} = useGetSearchQuery(queryOverride, parsedFilterData, {
refetchInterval: 5000,
+ enabled: !isWildcardQuery,
});
const { files: searchFiles, warnings: searchWarnings } =
searchData as SearchResult;
+ // Merge data from whichever source is active
+ const effectiveData: File[] = isWildcardQuery
+ ? (listFilesData?.files ?? [])
+ : searchFiles;
+ const isLoading = isWildcardQuery ? isListFilesLoading : isSearchLoading;
+ const error = isWildcardQuery ? listFilesError : searchError;
+ const isError = isWildcardQuery ? isListFilesError : isSearchError;
+
const isOpenragDocsRow = useCallback((file?: File) => {
return (
file?.connector_type === "openrag_docs" ||
@@ -312,7 +348,7 @@ function SearchPage() {
}
}, [isError, error]);
const fileResults = buildKnowledgeTableRows(
- searchFiles,
+ effectiveData,
taskFiles,
!!parsedFilterData,
);
@@ -836,7 +872,7 @@ function SearchPage() {
className="w-full overflow-auto border"
columnDefs={columnDefs as ColDef<File>[]}
defaultColDef={defaultColDef}
- loading={isSearchLoading || deleteDocumentMutation.isPending}
+ loading={isLoading || deleteDocumentMutation.isPending}
ref={gridRef}
theme={themeQuartz.withParams({ browserColorScheme: "inherit" })}
rowData={gridRows}
@@ -867,7 +903,7 @@ function SearchPage() {
className="w-full overflow-auto"
columnDefs={columnDefs as ColDef<File>[]}
defaultColDef={defaultColDef}
- loading={isSearchLoading || deleteDocumentMutation.isPending}
+ loading={isLoading || deleteDocumentMutation.isPending}
ref={gridRef}
theme={themeQuartz.withParams({ browserColorScheme: "inherit" })}
rowData={gridRows}
diff --git a/frontend/app/settings/_components/connector-card.tsx b/frontend/app/settings/_components/connector-card.tsx
index 820441b93..314284ffd 100644
--- a/frontend/app/settings/_components/connector-card.tsx
+++ b/frontend/app/settings/_components/connector-card.tsx
@@ -189,7 +189,7 @@ export default function ConnectorCard({
className={cn(
"w-full cursor-pointer",
isCloudBrand
- ? "rounded-none border-layer-contextual-foreground bg-layer-contextual text-layer-contextual-foreground hover:bg-primary hover:text-primary-foreground hover:border-primary"
+ ? "rounded-none border-primary bg-layer-contextual text-layer-contextual-foreground hover:bg-primary hover:text-primary-foreground hover:border-primary"
: "group-hover:bg-background group-hover:border-zinc-700 group-hover:text-primary",
)}
size="sm"
diff --git a/frontend/app/upload/[provider]/page.tsx b/frontend/app/upload/[provider]/page.tsx
index 4235c34ab..6d315d85e 100644
--- a/frontend/app/upload/[provider]/page.tsx
+++ b/frontend/app/upload/[provider]/page.tsx
@@ -1,7 +1,13 @@
"use client";
import { useQueryClient } from "@tanstack/react-query";
-import { AlertCircle, ArrowLeft, FolderOpen, RefreshCw } from "lucide-react";
+import {
+ AlertCircle,
+ ArrowLeft,
+ FileSearch,
+ FolderOpen,
+ RefreshCw,
+} from "lucide-react";
import { useParams, useRouter } from "next/navigation";
import { useState } from "react";
import { toast } from "sonner";
@@ -13,6 +19,7 @@ import { useS3BucketStatusQuery } from "@/app/api/queries/useS3BucketStatusQuery
import { type CloudFile, UnifiedCloudPicker } from "@/components/cloud-picker";
import { IngestSettings } from "@/components/cloud-picker/ingest-settings";
import { getIngestChunkSettingsError } from "@/components/cloud-picker/types";
+import { FileBrowserDialog } from "@/components/file-browser-dialog";
import { Button } from "@/components/ui/button";
import {
Tooltip,
@@ -58,6 +65,9 @@ function BucketView({
);
const [ingestSettings, setIngestSettings] = useSessionIngestSettings();
const [isSettingsOpen, setIsSettingsOpen] = useState(false);
+ const [browseDialogBucket, setBrowseDialogBucket] = useState<string | null>(
+ null,
+ );
const invalidate = () => {
queryClient.invalidateQueries({ queryKey: invalidateQueryKey });
@@ -218,6 +228,18 @@ function BucketView({
</p>
)}
</div>
+ <Button
+ variant="outline"
+ size="sm"
+ className="shrink-0"
+ onClick={(e) => {
+ e.stopPropagation();
+ setBrowseDialogBucket(bucket.name);
+ }}
+ >
+ <FileSearch size={14} className="mr-1.5" />
+ Browse Files
+ </Button>
</div>
</div>
);
@@ -256,6 +278,18 @@ function BucketView({
</Button>
</div>
</div>
+
+ {connector.connectionId && browseDialogBucket && (
+ <FileBrowserDialog
+ open
+ onOpenChange={(open) => {
+ if (!open) setBrowseDialogBucket(null);
+ }}
+ connectorType={connector.type}
+ connectionId={connector.connectionId}
+ buckets={[browseDialogBucket]}
+ />
+ )}
</>
);
}
diff --git a/frontend/components/file-browser-dialog.tsx b/frontend/components/file-browser-dialog.tsx
new file mode 100644
index 000000000..0681d339c
--- /dev/null
+++ b/frontend/components/file-browser-dialog.tsx
@@ -0,0 +1,289 @@
+"use client";
+
+import { Check, Download, Loader2, Search } from "lucide-react";
+import { useCallback, useMemo, useState } from "react";
+import { toast } from "sonner";
+import { useSyncConnector } from "@/app/api/mutations/useSyncConnector";
+import {
+ type RemoteFile,
+ useBrowseConnectionFiles,
+} from "@/app/api/queries/useBrowseConnectionFiles";
+import { Badge } from "./ui/badge";
+import { Button } from "./ui/button";
+import {
+ Dialog,
+ DialogContent,
+ DialogDescription,
+ DialogFooter,
+ DialogHeader,
+ DialogTitle,
+} from "./ui/dialog";
+import { Input } from "./ui/input";
+import { ScrollArea } from "./ui/scroll-area";
+
+interface FileBrowserDialogProps {
+ open: boolean;
+ onOpenChange: (open: boolean) => void;
+ connectorType: string;
+ connectionId: string;
+ buckets?: string[];
+}
+
+function formatFileSize(bytes: number): string {
+ if (bytes === 0) return "0 B";
+ const units = ["B", "KB", "MB", "GB"];
+ const i = Math.floor(Math.log(bytes) / Math.log(1024));
+ return `${(bytes / Math.pow(1024, i)).toFixed(i === 0 ? 0 : 1)} ${units[i]}`;
+}
+
+export function FileBrowserDialog({
+ open,
+ onOpenChange,
+ connectorType,
+ connectionId,
+ buckets,
+}: FileBrowserDialogProps) {
+ const [search, setSearch] = useState("");
+ const [selectedBucket, setSelectedBucket] = useState<string | undefined>(
+ buckets?.[0],
+ );
+ const [selectedFileIds, setSelectedFileIds] = useState<Set<string>>(
+ new Set(),
+ );
+
+ const syncMutation = useSyncConnector();
+
+ const { data, isLoading, error } = useBrowseConnectionFiles(
+ {
+ connectorType,
+ connectionId,
+ bucket: selectedBucket,
+ search: search || undefined,
+ maxFiles: 500,
+ },
+ { enabled: open },
+ );
+
+ const files = data?.files ?? [];
+
+ const toggleFile = useCallback((fileId: string) => {
+ setSelectedFileIds((prev) => {
+ const next = new Set(prev);
+ if (next.has(fileId)) {
+ next.delete(fileId);
+ } else {
+ next.add(fileId);
+ }
+ return next;
+ });
+ }, []);
+
+ const toggleAll = useCallback(() => {
+ const unIngestedFiles = files.filter((f) => !f.is_ingested);
+ if (selectedFileIds.size === unIngestedFiles.length) {
+ setSelectedFileIds(new Set());
+ } else {
+ setSelectedFileIds(new Set(unIngestedFiles.map((f) => f.id)));
+ }
+ }, [files, selectedFileIds.size]);
+
+ const selectedFiles = useMemo(
+ () => files.filter((f) => selectedFileIds.has(f.id)),
+ [files, selectedFileIds],
+ );
+
+ const handleIngest = useCallback(async () => {
+ if (selectedFiles.length === 0) return;
+
+ try {
+ await syncMutation.mutateAsync({
+ connectorType,
+ body: {
+ selected_files: selectedFiles.map((f) => ({
+ id: f.id,
+ name: f.name,
+ mimeType: "",
+ size: f.size,
+ })),
+ },
+ });
+
+ toast.success("Ingestion started", {
+ description: `${selectedFiles.length} file(s) queued for ingestion.`,
+ });
+
+ setSelectedFileIds(new Set());
+ onOpenChange(false);
+ } catch (err) {
+ toast.error("Ingestion failed", {
+ description: err instanceof Error ? err.message : "Unknown error",
+ });
+ }
+ }, [selectedFiles, connectorType, syncMutation, onOpenChange]);
+
+ const unIngestedCount = files.filter((f) => !f.is_ingested).length;
+
+ return (
+ <Dialog open={open} onOpenChange={onOpenChange}>
+ <DialogContent className="max-w-2xl max-h-[80vh] flex flex-col">
+ <DialogHeader>
+ <DialogTitle>Browse Files</DialogTitle>
+ <DialogDescription>
+ Select files to ingest from your {connectorType.replace("_", " ")}{" "}
+ connection.
+ {data && (
+ <span className="ml-1">
+ {data.total_ingested} of {data.total_remote} file(s) already
+ ingested.
+ </span>
+ )}
+ </DialogDescription>
+ </DialogHeader>
+
+ <div className="flex gap-2 items-center">
+ {buckets && buckets.length > 1 && (
+ <select
+ className="border rounded px-2 py-1.5 text-sm bg-background"
+ value={selectedBucket || ""}
+ onChange={(e) => {
+ setSelectedBucket(e.target.value || undefined);
+ setSelectedFileIds(new Set());
+ }}
+ >
+ {buckets.map((b) => (
+ <option key={b} value={b}>
+ {b}
+ </option>
+ ))}
+ </select>
+ )}
+ <div className="relative flex-1">
+ <Search className="absolute left-2.5 top-2.5 h-4 w-4 text-muted-foreground" />
+ <Input
+ placeholder="Search files..."
+ value={search}
+ onChange={(e) => setSearch(e.target.value)}
+ className="pl-9"
+ />
+ </div>
+ </div>
+
+ <ScrollArea className="flex-1 min-h-0 max-h-[400px] border rounded">
+ {isLoading ? (
+ <div className="flex items-center justify-center p-8">
+ <Loader2 className="h-6 w-6 animate-spin text-muted-foreground" />
+ <span className="ml-2 text-muted-foreground">
+ Loading files...
+ </span>
+ </div>
+ ) : error ? (
+ <div className="p-4 text-destructive text-sm">
+ Failed to load files:{" "}
+ {error instanceof Error ? error.message : "Unknown error"}
+ </div>
+ ) : files.length === 0 ? (
+ <div className="p-8 text-center text-muted-foreground text-sm">
+ No files found.
+ </div>
+ ) : (
+ <div className="divide-y">
+ {unIngestedCount > 0 && (
+ <div className="px-3 py-2 bg-muted/50 flex items-center gap-2 sticky top-0">
+ <input
+ type="checkbox"
+ checked={
+ selectedFileIds.size === unIngestedCount &&
+ unIngestedCount > 0
+ }
+ onChange={toggleAll}
+ className="h-4 w-4 rounded border-border"
+ />
+ <span className="text-xs text-muted-foreground">
+ {selectedFileIds.size > 0
+ ? `${selectedFileIds.size} selected`
+ : `Select all (${unIngestedCount})`}
+ </span>
+ </div>
+ )}
+ {files.map((file) => (
+ <FileRow
+ key={file.id}
+ file={file}
+ selected={selectedFileIds.has(file.id)}
+ onToggle={() => toggleFile(file.id)}
+ />
+ ))}
+ </div>
+ )}
+ </ScrollArea>
+
+ <DialogFooter>
+ <Button variant="outline" onClick={() => onOpenChange(false)}>
+ Cancel
+ </Button>
+ <Button
+ onClick={handleIngest}
+ disabled={selectedFiles.length === 0 || syncMutation.isPending}
+ >
+ {syncMutation.isPending ? (
+ <>
+ <Loader2 className="h-4 w-4 mr-2 animate-spin" />
+ Ingesting...
+ </>
+ ) : (
+ <>
+ <Download className="h-4 w-4 mr-2" />
+ Ingest{" "}
+ {selectedFiles.length > 0
+ ? `${selectedFiles.length} file(s)`
+ : "selected"}
+ </>
+ )}
+ </Button>
+ </DialogFooter>
+ </DialogContent>
+ </Dialog>
+ );
+}
+
+function FileRow({
+ file,
+ selected,
+ onToggle,
+}: {
+ file: RemoteFile;
+ selected: boolean;
+ onToggle: () => void;
+}) {
+ return (
+ <label
+ className={`flex items-center gap-3 px-3 py-2.5 cursor-pointer hover:bg-muted/30 transition-colors ${
+ file.is_ingested ? "opacity-60" : ""
+ }`}
+ >
+ <input
+ type="checkbox"
+ checked={selected || file.is_ingested}
+ disabled={file.is_ingested}
+ onChange={onToggle}
+ className="h-4 w-4 rounded border-border"
+ />
+ <div className="flex-1 min-w-0">
+ <div className="text-sm truncate font-medium">{file.name}</div>
+ <div className="text-xs text-muted-foreground flex gap-2">
+ {file.bucket && <span>{file.bucket}</span>}
+ <span>{formatFileSize(file.size)}</span>
+ {file.modified_time && (
+ <span>{new Date(file.modified_time).toLocaleDateString()}</span>
+ )}
+ </div>
+ </div>
+ {file.is_ingested && (
+ <Badge variant="secondary" className="text-xs flex-shrink-0">
+ <Check className="h-3 w-3 mr-1" />
+ Ingested
+ </Badge>
+ )}
+ </label>
+ );
+}
diff --git a/frontend/components/knowledge-batch-actions-bar.tsx b/frontend/components/knowledge-batch-actions-bar.tsx
index 71e998902..3d730d20d 100644
--- a/frontend/components/knowledge-batch-actions-bar.tsx
+++ b/frontend/components/knowledge-batch-actions-bar.tsx
@@ -17,6 +17,14 @@ export const KnowledgeBatchActionsBar = ({
const canDelete = canAny(["knowledge:delete:own", "knowledge:delete:any"]);
return (
<div className="flex h-12 w-full items-stretch bg-primary text-primary-foreground">
+ <button
+ type="button"
+ aria-label="Cancel selection"
+ onClick={onCancel}
+ className="flex h-full w-12 flex-shrink-0 items-center justify-center border-r border-primary-foreground/20 transition-colors hover:bg-primary-foreground/10"
+ >
+ <X className="h-4 w-4" />
+ </button>
<span className="flex items-center px-4 text-sm font-medium">
{selectedCount} item{selectedCount !== 1 ? "s" : ""} selected
</span>
@@ -32,17 +40,9 @@ export const KnowledgeBatchActionsBar = ({
}
className="flex h-full items-center px-4 text-sm font-medium transition-colors hover:bg-primary-foreground/10 disabled:opacity-50 disabled:cursor-not-allowed"
>
+ <Trash2 className="h-4 w-4" />
Delete
</button>
- <span className="self-center h-4 w-px bg-primary-foreground" />
- <button
- type="button"
- aria-label="Cancel selection"
- onClick={onCancel}
- className="flex h-full w-12 flex-shrink-0 items-center justify-center transition-colors hover:bg-primary-foreground/10"
- >
- <X className="h-4 w-4" />
- </button>
</div>
</div>
);
diff --git a/src/api/acl.py b/src/api/acl.py
new file mode 100644
index 000000000..2b9180f7c
--- /dev/null
+++ b/src/api/acl.py
@@ -0,0 +1,38 @@
+from fastapi import Depends
+from fastapi.responses import JSONResponse
+
+from config.settings import get_index_name
+from dependencies import get_current_user, get_session_manager
+from session_manager import User
+
+
+async def get_document_acl(
+ filename: str,
+ session_manager=Depends(get_session_manager),
+ user: User = Depends(get_current_user),
+):
+ """GET /documents/acl?filename=... — return ACL for a document."""
+ opensearch_client = session_manager.get_user_opensearch_client(user.user_id, user.jwt_token)
+ response = await opensearch_client.search(
+ index=get_index_name(),
+ body={
+ "query": {"term": {"filename": filename}},
+ "size": 1,
+ "_source": ["owner", "allowed_users", "allowed_groups"],
+ },
+ )
+ hits = response.get("hits", {}).get("hits", [])
+ if not hits:
+ return JSONResponse(
+ {"error": f"Document '{filename}' not found"},
+ status_code=404,
+ )
+
+ source = hits[0]["_source"]
+ return JSONResponse(
+ {
+ "owner": source.get("owner"),
+ "allowed_users": source.get("allowed_users", []),
+ "allowed_groups": source.get("allowed_groups", []),
+ }
+ )
diff --git a/src/api/connectors.py b/src/api/connectors.py
index 2fceae4d5..e4213d828 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -28,59 +28,46 @@ async def get_synced_file_ids_for_connector(
"""
Query OpenSearch for unique document_id values where connector_type matches.
Returns tuple of (file_ids, filenames) - use file_ids if available, else filenames as fallback.
-
+
Note: Langflow-ingested files may not have document_id stored. In that case,
filenames are returned for filename-based filtering during sync.
"""
try:
opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
-
+
# Query for both document_id and filename aggregations
query_body = {
"size": 0,
- "query": {
- "term": {
- "connector_type": connector_type
- }
- },
+ "query": {"term": {"connector_type": connector_type}},
"aggs": {
- "unique_document_ids": {
- "terms": {
- "field": "document_id",
- "size": 10000
- }
- },
- "unique_filenames": {
- "terms": {
- "field": "filename",
- "size": 10000
- }
- }
- }
+ "unique_document_ids": {"terms": {"field": "document_id", "size": 10000}},
+ "unique_filenames": {"terms": {"field": "filename", "size": 10000}},
+ },
}
-
- result = await opensearch_client.search(
- index=get_index_name(),
- body=query_body
- )
-
+
+ result = await opensearch_client.search(index=get_index_name(), body=query_body)
+
# Get document_ids (preferred - these are the actual connector file IDs)
- doc_id_buckets = result.get("aggregations", {}).get("unique_document_ids", {}).get("buckets", [])
+ doc_id_buckets = (
+ result.get("aggregations", {}).get("unique_document_ids", {}).get("buckets", [])
+ )
file_ids = [bucket["key"] for bucket in doc_id_buckets if bucket["key"]]
-
+
# Get filenames as fallback
- filename_buckets = result.get("aggregations", {}).get("unique_filenames", {}).get("buckets", [])
+ filename_buckets = (
+ result.get("aggregations", {}).get("unique_filenames", {}).get("buckets", [])
+ )
filenames = [bucket["key"] for bucket in filename_buckets if bucket["key"]]
-
+
logger.debug(
"Found synced files for connector",
connector_type=connector_type,
file_ids_count=len(file_ids),
filenames_count=len(filenames),
)
-
+
return file_ids, filenames
-
+
except Exception as e:
logger.error(
"Failed to get synced file IDs",
@@ -90,7 +77,6 @@ async def get_synced_file_ids_for_connector(
return [], []
-
class ConnectorSyncBody(BaseModel):
max_files: Optional[int] = None
selected_files: Optional[List[Any]] = None
@@ -140,7 +126,9 @@ async def connector_sync(
file_infos = selected_files_raw
try:
- await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_START)
+ await TelemetryClient.send_event(
+ Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_START
+ )
logger.debug(
"Starting connector sync",
connector_type=connector_type,
@@ -201,10 +189,11 @@ async def connector_sync(
"Starting sync with working connection",
connection_id=working_connection.connection_id,
)
-
+
if selected_files:
# Explicit files selected (e.g., from file picker) - sync those specific files
from .documents import _ensure_index_exists
+
await _ensure_index_exists(jwt_token)
task_id = await connector_service.sync_specific_files(
working_connection.connection_id,
@@ -242,7 +231,10 @@ async def connector_sync(
if not all_file_ids:
return JSONResponse(
- {"status": "no_files", "message": "No files found in the selected buckets."},
+ {
+ "status": "no_files",
+ "message": "No files found in the selected buckets.",
+ },
status_code=200,
)
task_id = await connector_service.sync_specific_files(
@@ -308,7 +300,9 @@ async def connector_sync(
filename_filter=set(existing_filenames),
)
task_ids = [task_id]
- await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_COMPLETE)
+ await TelemetryClient.send_event(
+ Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_COMPLETE
+ )
return JSONResponse(
{
"task_ids": task_ids,
@@ -321,7 +315,9 @@ async def connector_sync(
except Exception as e:
logger.error("Connector sync failed", error=str(e))
- await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_FAILED)
+ await TelemetryClient.send_event(
+ Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_FAILED
+ )
return JSONResponse({"error": f"Sync failed: {str(e)}"}, status_code=500)
@@ -340,25 +336,31 @@ async def connector_status(
# Get the connector for each connection and verify authentication
connection_details = {}
verified_active_connections = []
-
+
for connection in connections:
try:
connector = await connector_service._get_connector(connection.connection_id)
if connector is not None:
# Actually verify the connection by trying to authenticate
is_authenticated = await connector.authenticate()
-
+
# Get base URL if available (for SharePoint/OneDrive connectors)
base_url = None
- if hasattr(connector, 'base_url'):
+ if hasattr(connector, "base_url"):
base_url = connector.base_url
- logger.debug(f"connector_status: Got base_url from connector.base_url: {base_url}")
- elif hasattr(connector, 'sharepoint_url'):
+ logger.debug(
+ f"connector_status: Got base_url from connector.base_url: {base_url}"
+ )
+ elif hasattr(connector, "sharepoint_url"):
base_url = connector.sharepoint_url # Backward compatibility
- logger.debug(f"connector_status: Got base_url from connector.sharepoint_url: {base_url}")
+ logger.debug(
+ f"connector_status: Got base_url from connector.sharepoint_url: {base_url}"
+ )
else:
- logger.debug(f"connector_status: Connector has no base_url or sharepoint_url attribute")
-
+ logger.debug(
+ f"connector_status: Connector has no base_url or sharepoint_url attribute"
+ )
+
connection_details[connection.connection_id] = {
"client_id": connector.get_client_id(),
"is_authenticated": is_authenticated,
@@ -397,8 +399,13 @@ async def connector_status(
"connection_id": conn.connection_id,
"name": conn.name,
"client_id": connection_details.get(conn.connection_id, {}).get("client_id"),
- "is_active": conn.is_active and connection_details.get(conn.connection_id, {}).get("is_authenticated", False),
- "is_authenticated": connection_details.get(conn.connection_id, {}).get("is_authenticated", False),
+ "is_active": conn.is_active
+ and connection_details.get(conn.connection_id, {}).get(
+ "is_authenticated", False
+ ),
+ "is_authenticated": connection_details.get(conn.connection_id, {}).get(
+ "is_authenticated", False
+ ),
"base_url": connection_details.get(conn.connection_id, {}).get("base_url"),
"created_at": conn.created_at.isoformat(),
"last_sync": conn.last_sync.isoformat() if conn.last_sync else None,
@@ -428,10 +435,10 @@ async def connector_webhook(
config=temp_config,
)
try:
- await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_WEBHOOK_RECV)
- temp_connector = connector_service.connection_manager._create_connector(
- temp_connection
+ await TelemetryClient.send_event(
+ Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_WEBHOOK_RECV
)
+ temp_connector = connector_service.connection_manager._create_connector(temp_connection)
validation_response = temp_connector.handle_webhook_validation(
request.method, dict(request.headers), dict(request.query_params)
)
@@ -466,32 +473,22 @@ async def connector_webhook(
# Extract channel/subscription ID using connector-specific method
try:
- temp_connector = connector_service.connection_manager._create_connector(
- temp_connection
- )
+ temp_connector = connector_service.connection_manager._create_connector(temp_connection)
channel_id = temp_connector.extract_webhook_channel_id(payload, headers)
except (NotImplementedError, ValueError):
channel_id = None
if not channel_id:
- logger.warning(
- "No channel ID found in webhook", connector_type=connector_type
- )
+ logger.warning("No channel ID found in webhook", connector_type=connector_type)
return JSONResponse({"status": "ignored", "reason": "no_channel_id"})
# Find the specific connection for this webhook
- connection = (
- await connector_service.connection_manager.get_connection_by_webhook_id(
- channel_id
- )
+ connection = await connector_service.connection_manager.get_connection_by_webhook_id(
+ channel_id
)
if not connection or not connection.is_active:
- logger.info(
- "Unknown webhook channel, will auto-expire", channel_id=channel_id
- )
- return JSONResponse(
- {"status": "ignored_unknown_channel", "channel_id": channel_id}
- )
+ logger.info("Unknown webhook channel, will auto-expire", channel_id=channel_id)
+ return JSONResponse({"status": "ignored_unknown_channel", "channel_id": channel_id})
# Process webhook for the specific connection
try:
@@ -502,9 +499,7 @@ async def connector_webhook(
"Could not get connector for connection",
connection_id=connection.connection_id,
)
- return JSONResponse(
- {"status": "error", "reason": "connector_not_found"}
- )
+ return JSONResponse({"status": "error", "reason": "connector_not_found"})
# Let the connector handle the webhook and return affected file IDs
affected_files = await connector.handle_webhook(payload)
@@ -575,10 +570,11 @@ async def connector_webhook(
except Exception as e:
logger.error("Webhook processing failed", error=str(e))
- await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_WEBHOOK_FAILED)
- return JSONResponse(
- {"error": f"Webhook processing failed: {str(e)}"}, status_code=500
+ await TelemetryClient.send_event(
+ Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_WEBHOOK_FAILED
)
+ return JSONResponse({"error": f"Webhook processing failed: {str(e)}"}, status_code=500)
+
async def connector_disconnect(
connector_type: str,
@@ -605,7 +601,7 @@ async def connector_disconnect(
try:
# Get the connector to cleanup any subscriptions
connector = await connector_service._get_connector(connection.connection_id)
- if connector and hasattr(connector, 'cleanup_subscription'):
+ if connector and hasattr(connector, "cleanup_subscription"):
subscription_id = connection.config.get("webhook_channel_id")
if subscription_id:
try:
@@ -659,6 +655,7 @@ async def connector_disconnect(
# ---------------------------------------------------------------------------
+
async def sync_all_connectors(
connector_service=Depends(get_connector_service),
session_manager=Depends(get_session_manager),
@@ -668,12 +665,14 @@ async def sync_all_connectors(
Sync files from all active cloud connector connections.
"""
try:
- await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_START)
+ await TelemetryClient.send_event(
+ Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_START
+ )
jwt_token = user.jwt_token
# Cloud connector types to sync
cloud_connector_types = ["google_drive", "onedrive", "sharepoint", "ibm_cos", "aws_s3"]
-
+
all_task_ids = []
synced_connectors = []
skipped_connectors = []
@@ -688,7 +687,7 @@ async def sync_all_connectors(
session_manager=session_manager,
jwt_token=jwt_token,
)
-
+
if not existing_file_ids and not existing_filenames:
logger.debug(
"No existing files in OpenSearch for connector type, skipping",
@@ -760,14 +759,16 @@ async def sync_all_connectors(
jwt_token=jwt_token,
filename_filter=set(existing_filenames),
)
-
+
all_task_ids.append(task_id)
synced_connectors.append(connector_type)
logger.info(
"Started sync for connector type",
connector_type=connector_type,
task_id=task_id,
- file_count=len(existing_file_ids) if existing_file_ids else len(existing_filenames),
+ file_count=len(existing_file_ids)
+ if existing_file_ids
+ else len(existing_filenames),
)
except Exception as e:
@@ -793,7 +794,9 @@ async def sync_all_connectors(
status_code=404,
)
- await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_COMPLETE)
+ await TelemetryClient.send_event(
+ Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_COMPLETE
+ )
return JSONResponse(
{
"task_ids": all_task_ids,
@@ -808,7 +811,9 @@ async def sync_all_connectors(
except Exception as e:
logger.error("Sync all connectors failed", error=str(e))
- await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_FAILED)
+ await TelemetryClient.send_event(
+ Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_FAILED
+ )
return JSONResponse({"error": f"Sync failed: {str(e)}"}, status_code=500)
@@ -832,7 +837,9 @@ async def connector_token(
connector = await connector_service._get_connector(connection_id)
if not connector:
return JSONResponse(
- {"error": f"Connector not available - authentication may have failed for {url_connector_type}"},
+ {
+ "error": f"Connector not available - authentication may have failed for {url_connector_type}"
+ },
status_code=404,
)
@@ -864,7 +871,10 @@ async def connector_token(
try:
if connector.oauth.creds.expiry:
import time
- expires_in = max(0, int(connector.oauth.creds.expiry.timestamp() - time.time()))
+
+ expires_in = max(
+ 0, int(connector.oauth.creds.expiry.timestamp() - time.time())
+ )
except Exception:
expires_in = None
@@ -911,12 +921,114 @@ async def connector_token(
return JSONResponse({"access_token": access_token, "expires_in": None})
except ValueError as e:
# Typical when acquire_token_silent fails (e.g., needs re-auth)
- return JSONResponse({"error": f"Failed to get access token: {str(e)}"}, status_code=401)
+ return JSONResponse(
+ {"error": f"Failed to get access token: {str(e)}"}, status_code=401
+ )
except Exception as e:
return JSONResponse({"error": f"Authentication error: {str(e)}"}, status_code=500)
- return JSONResponse({"error": "Token not available for this connector type"}, status_code=400)
+ return JSONResponse(
+ {"error": "Token not available for this connector type"}, status_code=400
+ )
except Exception as e:
logger.error("Error getting connector token", exc_info=True)
return JSONResponse({"error": str(e)}, status_code=500)
+
+
+async def browse_connection_files(
+ connector_type: str,
+ connection_id: str,
+ connector_service=Depends(get_connector_service),
+ session_manager=Depends(get_session_manager),
+ user: User = Depends(get_current_user),
+ bucket: Optional[str] = None,
+ search: Optional[str] = None,
+ page_token: Optional[str] = None,
+ max_files: int = 100,
+):
+ """
+ Browse remote files in a connector with ingestion status.
+
+ Lists files from the remote source (e.g., S3 bucket) and marks each
+ as ingested or not by cross-referencing with OpenSearch.
+ """
+ try:
+ connector = await connector_service.get_connector(connection_id)
+ if not connector:
+ return JSONResponse(
+ {"error": "Connection not found or connector unavailable"},
+ status_code=404,
+ )
+
+ if not await connector.authenticate():
+ return JSONResponse(
+ {"error": "Connector authentication failed"},
+ status_code=401,
+ )
+
+ # Temporarily override bucket filter if specified
+ original_buckets = None
+ if bucket and hasattr(connector, "bucket_names"):
+ original_buckets = connector.bucket_names
+ connector.bucket_names = [bucket]
+
+ try:
+ files_result = await connector.list_files(page_token=page_token, max_files=max_files)
+ finally:
+ if original_buckets is not None:
+ connector.bucket_names = original_buckets
+
+ remote_files = files_result.get("files", [])
+ next_page_token = files_result.get("next_page_token")
+
+ # Filter by filename search if provided
+ if search:
+ search_lower = search.lower()
+ remote_files = [f for f in remote_files if search_lower in f.get("name", "").lower()]
+
+ # Get already-ingested file IDs from OpenSearch
+ ingested_ids, ingested_filenames = await get_synced_file_ids_for_connector(
+ connector_type=connector_type,
+ user_id=user.user_id,
+ session_manager=session_manager,
+ jwt_token=user.jwt_token,
+ )
+ ingested_set = set(ingested_ids) | set(ingested_filenames)
+
+ # Merge ingestion status into remote file list
+ enriched_files = []
+ for f in remote_files:
+ is_ingested = f.get("id", "") in ingested_set or f.get("name", "") in ingested_set
+ enriched_files.append(
+ {
+ "id": f.get("id", ""),
+ "name": f.get("name", ""),
+ "bucket": f.get("bucket", ""),
+ "key": f.get("key", ""),
+ "size": f.get("size", 0),
+ "modified_time": f.get("modified_time", ""),
+ "is_ingested": is_ingested,
+ }
+ )
+
+ return JSONResponse(
+ {
+ "files": enriched_files,
+ "next_page_token": next_page_token,
+ "total_remote": len(enriched_files),
+ "total_ingested": sum(1 for f in enriched_files if f["is_ingested"]),
+ }
+ )
+
+ except Exception as e:
+ logger.error(
+ "Failed to browse connection files",
+ connector_type=connector_type,
+ connection_id=connection_id,
+ error=str(e),
+ )
+ return JSONResponse(
+ {"error": f"Failed to browse files: {str(e)}"},
+ status_code=500,
+ )
diff --git a/src/api/documents.py b/src/api/documents.py
index 416e146bd..ce9b6275b 100644
--- a/src/api/documents.py
+++ b/src/api/documents.py
@@ -37,9 +37,32 @@ async def delete_documents_by_filename_core(
)
try:
- opensearch_client = session_manager.get_user_opensearch_client(
- user_id, jwt_token
+ opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
+
+ # Owner check: only the document owner may delete
+ check = await opensearch_client.search(
+ index=get_index_name(),
+ body={
+ "query": {"term": {"filename": normalized_filename}},
+ "size": 1,
+ "_source": ["owner"],
+ },
)
+ hits = check.get("hits", {}).get("hits", [])
+ if hits:
+ doc_owner = hits[0]["_source"].get("owner")
+ if doc_owner and doc_owner != user_id:
+ return (
+ {
+ "success": False,
+ "deleted_chunks": 0,
+ "filename": normalized_filename,
+ "message": None,
+ "error": "Access denied: only the document owner can delete this file",
+ },
+ 403,
+ )
+
delete_query = build_filename_delete_body(normalized_filename)
result = await opensearch_client.delete_by_query(
index=get_index_name(),
@@ -122,9 +145,7 @@ async def check_filename_exists(
jwt_token = user.jwt_token
try:
- opensearch_client = session_manager.get_user_opensearch_client(
- user.user_id, jwt_token
- )
+ opensearch_client = session_manager.get_user_opensearch_client(user.user_id, jwt_token)
from utils.opensearch_queries import build_filename_search_body
from utils.file_utils import get_filename_aliases
@@ -139,10 +160,7 @@ async def check_filename_exists(
try:
for candidate in candidate_filenames:
search_body = build_filename_search_body(candidate, size=1, source=["filename"])
- response = await opensearch_client.search(
- index=get_index_name(),
- body=search_body
- )
+ response = await opensearch_client.search(index=get_index_name(), body=search_body)
hits = response.get("hits", {}).get("hits", [])
if hits:
exists = True
@@ -160,7 +178,9 @@ async def check_filename_exists(
logger.error("Error checking filename existence", filename=filename, error=str(e))
error_str = str(e)
if "AuthenticationException" in error_str:
- return JSONResponse({"error": "Access denied: insufficient permissions"}, status_code=403)
+ return JSONResponse(
+ {"error": "Access denied: insufficient permissions"}, status_code=403
+ )
else:
return JSONResponse({"error": str(e)}, status_code=500)
@@ -169,9 +189,9 @@ async def delete_documents_by_filename(
body: DeleteDocumentBody,
session_manager=Depends(get_session_manager),
user: User = Depends(require_permission("knowledge:delete:own")),
- ):
+):
"""Delete all documents with a specific filename"""
- payload, status_code =await delete_documents_by_filename_core(
+ payload, status_code = await delete_documents_by_filename_core(
filename=body.filename,
session_manager=session_manager,
user_id=user.user_id,
diff --git a/src/api/files.py b/src/api/files.py
new file mode 100644
index 000000000..b88fcf636
--- /dev/null
+++ b/src/api/files.py
@@ -0,0 +1,89 @@
+"""
+File listing and search API endpoints.
+
+Provides server-side file-level views over the chunk-based OpenSearch index,
+replacing the client-side chunk-to-file aggregation pattern.
+"""
+
+from typing import Optional
+
+from fastapi import Depends, Query
+from fastapi.responses import JSONResponse
+
+from dependencies import get_session_manager, get_current_user
+from session_manager import User
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+def _get_file_service(session_manager=Depends(get_session_manager)):
+ from services.file_service import FileService
+
+ return FileService(session_manager=session_manager)
+
+
+async def list_files(
+ page: int = Query(1, ge=1, description="Page number"),
+ page_size: int = Query(25, ge=1, le=100, description="Items per page"),
+ sort_by: str = Query("filename", description="Sort field"),
+ sort_order: str = Query("asc", regex="^(asc|desc)$", description="Sort order"),
+ connector_type: Optional[str] = Query(None, description="Filter by connector type"),
+ mimetype: Optional[str] = Query(None, description="Filter by MIME type"),
+ owner: Optional[str] = Query(None, description="Filter by owner"),
+ search: Optional[str] = Query(None, description="Search filename"),
+ file_service=Depends(_get_file_service),
+ user: User = Depends(get_current_user),
+):
+ """List ingested files with pagination, filtering, and sorting."""
+ try:
+ result = await file_service.list_files(
+ user_id=user.user_id,
+ jwt_token=user.jwt_token,
+ page=page,
+ page_size=page_size,
+ sort_by=sort_by,
+ sort_order=sort_order,
+ connector_type=connector_type,
+ mimetype=mimetype,
+ owner=owner,
+ search=search,
+ )
+ return JSONResponse(result)
+ except Exception as e:
+ logger.error("Failed to list files", error=str(e))
+ return JSONResponse(
+ {"error": "Failed to list files", "detail": str(e)},
+ status_code=500,
+ )
+
+
+async def search_files(
+ q: str = Query(..., min_length=1, description="Search query"),
+ page: int = Query(1, ge=1, description="Page number"),
+ page_size: int = Query(25, ge=1, le=100, description="Items per page"),
+ connector_type: Optional[str] = Query(None, description="Filter by connector type"),
+ mimetype: Optional[str] = Query(None, description="Filter by MIME type"),
+ owner: Optional[str] = Query(None, description="Filter by owner"),
+ file_service=Depends(_get_file_service),
+ user: User = Depends(get_current_user),
+):
+ """Search files by name with fuzzy/partial matching."""
+ try:
+ result = await file_service.search_files(
+ user_id=user.user_id,
+ jwt_token=user.jwt_token,
+ query=q,
+ page=page,
+ page_size=page_size,
+ connector_type=connector_type,
+ mimetype=mimetype,
+ owner=owner,
+ )
+ return JSONResponse(result)
+ except Exception as e:
+ logger.error("Failed to search files", error=str(e))
+ return JSONResponse(
+ {"error": "Failed to search files", "detail": str(e)},
+ status_code=500,
+ )
diff --git a/src/app/routes/internal.py b/src/app/routes/internal.py
index 78bc8ec63..c1de1914f 100644
--- a/src/app/routes/internal.py
+++ b/src/app/routes/internal.py
@@ -14,6 +14,7 @@ from api import (
connectors,
docling,
documents,
+ files,
flows,
knowledge_filter,
langflow_files,
@@ -89,6 +90,10 @@ def register_internal_routes(app: FastAPI):
# Search endpoint
app.add_api_route("/search", search.search, methods=["POST"], tags=["internal"])
+ # File listing/search endpoints
+ app.add_api_route("/files", files.list_files, methods=["GET"], tags=["internal"])
+ app.add_api_route("/files/search", files.search_files, methods=["GET"], tags=["internal"])
+
# Knowledge Filter endpoints
app.add_api_route(
"/knowledge-filter",
diff --git a/src/connectors/aws_s3/support.py b/src/connectors/aws_s3/support.py
index ccaa069b7..1dbbef99e 100644
--- a/src/connectors/aws_s3/support.py
+++ b/src/connectors/aws_s3/support.py
@@ -24,14 +24,13 @@ def build_s3_config(
({}, error_message) on validation failure
"""
access_key = (
- body.access_key
- or os.getenv("AWS_ACCESS_KEY_ID")
- or existing_config.get("access_key")
+ body.access_key or os.getenv("AWS_ACCESS_KEY_ID") or existing_config.get("access_key")
)
secret_key = (
- body.secret_key
- or os.getenv("AWS_SECRET_ACCESS_KEY")
- or existing_config.get("secret_key")
+ body.secret_key or os.getenv("AWS_SECRET_ACCESS_KEY") or existing_config.get("secret_key")
+ )
+ session_token = (
+ body.session_token or os.getenv("AWS_SESSION_TOKEN") or existing_config.get("session_token")
)
if not access_key or not secret_key:
@@ -41,11 +40,19 @@ def build_s3_config(
"access_key": access_key.strip(),
"secret_key": secret_key.strip(),
}
+ if session_token:
+ conn_config["session_token"] = session_token.strip()
if body.endpoint_url:
conn_config["endpoint_url"] = body.endpoint_url.strip()
+ elif existing_config.get("endpoint_url"):
+ conn_config["endpoint_url"] = existing_config["endpoint_url"]
if body.region:
conn_config["region"] = body.region.strip()
+ elif existing_config.get("region"):
+ conn_config["region"] = existing_config["region"]
if body.bucket_names is not None:
conn_config["bucket_names"] = body.bucket_names
+ elif existing_config.get("bucket_names") is not None:
+ conn_config["bucket_names"] = existing_config["bucket_names"]
return conn_config, None
diff --git a/src/connectors/connection_manager.py b/src/connectors/connection_manager.py
index 8646d72aa..79695bf45 100644
--- a/src/connectors/connection_manager.py
+++ b/src/connectors/connection_manager.py
@@ -39,9 +39,12 @@ class ConnectionConfig:
class ConnectionManager:
"""Manages multiple connector connections with persistence"""
- def __init__(self, connections_file: str = None):
- from config.paths import get_data_file
- self.connections_file = Path(connections_file or get_data_file("connections.json"))
+ def __init__(self, connections_file: Optional[str] = None):
+ if connections_file is None:
+ data_dir = Path(os.getenv("OPENRAG_DATA_PATH", "data"))
+ self.connections_file = data_dir / "connections.json"
+ else:
+ self.connections_file = Path(connections_file)
# Ensure data directory exists
self.connections_file.parent.mkdir(parents=True, exist_ok=True)
self.connections: Dict[str, ConnectionConfig] = {}
@@ -50,15 +53,24 @@ class ConnectionManager:
async def load_connections(self):
"""Load connections from persistent storage"""
from utils.encryption import decrypt_secret, get_master_secret
-
+
needs_encryption_upgrade = False
decryption_failed = False
secret_keys = {
- "api_key", "hmac_secret_key", "secret_key", "client_secret",
- "aws_secret_access_key", "ibm_api_key", "access_token", "refresh_token",
- "access_key", "hmac_access_key", "service_instance_id","basic_credentials"
+ "api_key",
+ "hmac_secret_key",
+ "secret_key",
+ "client_secret",
+ "aws_secret_access_key",
+ "ibm_api_key",
+ "access_token",
+ "refresh_token",
+ "access_key",
+ "hmac_access_key",
+ "service_instance_id",
+ "basic_credentials",
}
-
+
if self.connections_file.exists():
async with aiofiles.open(self.connections_file, "r") as f:
data = json.loads(await f.read())
@@ -70,27 +82,25 @@ class ConnectionManager:
if isinstance(v, dict) and v.get("algorithm") == "AES-256-GCM":
try:
tenant = conn_data.get("user_id") or "openrag"
- conn_data["config"][k] = decrypt_secret(v, expected_tenant_id=tenant)
+ conn_data["config"][k] = decrypt_secret(
+ v, expected_tenant_id=tenant
+ )
except ValueError as e:
logger.error(f"Failed to decrypt connection secret {k}: {e}")
decryption_failed = True
elif k in secret_keys and isinstance(v, str) and v:
if get_master_secret() is not None:
needs_encryption_upgrade = True
-
+
# Convert datetime strings back to datetime objects
if conn_data.get("created_at"):
- conn_data["created_at"] = datetime.fromisoformat(
- conn_data["created_at"]
- )
+ conn_data["created_at"] = datetime.fromisoformat(conn_data["created_at"])
if conn_data.get("last_sync"):
- conn_data["last_sync"] = datetime.fromisoformat(
- conn_data["last_sync"]
- )
+ conn_data["last_sync"] = datetime.fromisoformat(conn_data["last_sync"])
config = ConnectionConfig(**conn_data)
self.connections[config.connection_id] = config
-
+
if needs_encryption_upgrade:
if decryption_failed:
logger.warning(
@@ -111,24 +121,34 @@ class ConnectionManager:
async def save_connections(self):
"""Save connections to persistent storage"""
from utils.encryption import encrypt_secret
+
secret_keys = {
- "api_key", "hmac_secret_key", "secret_key", "client_secret",
- "aws_secret_access_key", "ibm_api_key", "access_token", "refresh_token",
- "access_key", "hmac_access_key", "service_instance_id", "basic_credentials"
+ "api_key",
+ "hmac_secret_key",
+ "secret_key",
+ "client_secret",
+ "aws_secret_access_key",
+ "ibm_api_key",
+ "access_token",
+ "refresh_token",
+ "access_key",
+ "hmac_access_key",
+ "service_instance_id",
+ "basic_credentials",
}
-
+
data = {"connections": []}
for config in self.connections.values():
conn_data = asdict(config)
-
+
# Encrypt sensitive fields in config
if "config" in conn_data and isinstance(conn_data["config"], dict):
for k, v in conn_data["config"].items():
if k in secret_keys and isinstance(v, str):
tenant_id = conn_data.get("user_id") or "openrag"
conn_data["config"][k] = encrypt_secret(v, tenant_id=tenant_id)
-
+
# Convert datetime objects to strings
if conn_data.get("created_at"):
conn_data["created_at"] = conn_data["created_at"].isoformat()
@@ -185,57 +205,63 @@ class ConnectionManager:
async def cleanup_duplicate_connections(self, remove_duplicates=False):
"""
- Clean up duplicate connections, keeping only the most recent connection
+ Clean up duplicate connections, keeping only the most recent connection
per provider per user
-
+
Args:
remove_duplicates: If True, physically removes duplicates from connections.json
If False (default), just deactivates them
"""
logger.info("Starting cleanup of duplicate connections")
-
+
# Group connections by (connector_type, user_id)
grouped_connections = {}
-
+
for connection_id, connection in self.connections.items():
if not connection.is_active:
continue # Skip inactive connections
-
+
key = (connection.connector_type, connection.user_id)
-
+
if key not in grouped_connections:
grouped_connections[key] = []
-
+
grouped_connections[key].append((connection_id, connection))
-
+
# For each group, keep only the most recent connection
connections_to_remove = []
-
+
for (connector_type, user_id), connections in grouped_connections.items():
if len(connections) <= 1:
continue # No duplicates
-
- logger.info(f"Found {len(connections)} duplicate connections for {connector_type}, user {user_id}")
-
+
+ logger.info(
+ f"Found {len(connections)} duplicate connections for {connector_type}, user {user_id}"
+ )
+
# Sort by created_at, keep the most recent
connections.sort(key=lambda x: x[1].created_at, reverse=True)
-
+
# Keep the first (most recent), remove/deactivate the rest
for connection_id, connection in connections[1:]:
connections_to_remove.append((connection_id, connection))
- logger.info(f"Marking connection {connection_id} for {'removal' if remove_duplicates else 'deactivation'}")
-
+ logger.info(
+ f"Marking connection {connection_id} for {'removal' if remove_duplicates else 'deactivation'}"
+ )
+
# Remove or deactivate duplicate connections
for connection_id, connection in connections_to_remove:
if remove_duplicates:
await self.delete_connection(connection_id) # Handles token cleanup
else:
await self.deactivate_connection(connection_id)
-
+
action = "Removed" if remove_duplicates else "Deactivated"
- logger.info(f"Cleanup complete. {action} {len(connections_to_remove)} duplicate connections")
+ logger.info(
+ f"Cleanup complete. {action} {len(connections_to_remove)} duplicate connections"
+ )
return len(connections_to_remove)
-
+
async def update_connection(
self,
connection_id: str,
@@ -285,10 +311,10 @@ class ConnectionManager:
user_id: Optional[str] = None,
) -> str:
"""Create a new connection configuration, ensuring only one per provider per user"""
-
+
# Check if we already have an active connection for this provider and user
existing_connection = await self._get_existing_connection(connector_type, user_id)
-
+
if existing_connection:
# Check if the existing connection has a valid token
try:
@@ -296,26 +322,25 @@ class ConnectionManager:
if await connector.authenticate():
logger.info(
f"Using existing valid connection for {connector_type}",
- connection_id=existing_connection.connection_id
+ connection_id=existing_connection.connection_id,
)
# Update the existing connection with new config if needed
if config != existing_connection.config:
logger.info("Updating existing connection config")
await self.update_connection(
- existing_connection.connection_id,
- config=config
+ existing_connection.connection_id, config=config
)
return existing_connection.connection_id
except Exception as e:
logger.warning(
f"Existing connection authentication failed: {e}",
- connection_id=existing_connection.connection_id
+ connection_id=existing_connection.connection_id,
)
# If authentication fails, we'll create a new connection and clean up the old one
-
+
# Create new connection
connection_id = str(uuid.uuid4())
-
+
connection_config = ConnectionConfig(
connection_id=connection_id,
connector_type=connector_type,
@@ -325,10 +350,10 @@ class ConnectionManager:
)
self.connections[connection_id] = connection_config
-
+
# Clean up duplicates (will keep the newest, which is the one we just created)
await self.cleanup_duplicate_connections(remove_duplicates=True)
-
+
await self.save_connections()
return connection_id
@@ -368,10 +393,7 @@ class ConnectionManager:
connector = self.active_connectors[connection_id]
# Try to cleanup subscriptions if applicable
try:
- if (
- hasattr(connector, "webhook_channel_id")
- and connector.webhook_channel_id
- ):
+ if hasattr(connector, "webhook_channel_id") and connector.webhook_channel_id:
await connector.cleanup_subscription(connector.webhook_channel_id)
except:
pass # Best effort cleanup
@@ -385,7 +407,7 @@ class ConnectionManager:
async def get_connector(self, connection_id: str) -> Optional[BaseConnector]:
"""Get an active connector instance"""
logger.debug(f"Getting connector for connection_id: {connection_id}")
-
+
# Return cached connector if available
if connection_id in self.active_connectors:
connector = self.active_connectors[connection_id]
@@ -405,11 +427,11 @@ class ConnectionManager:
logger.debug(f"Creating connector for {connection_config.connector_type}")
connector = self._create_connector(connection_config)
-
+
logger.debug(f"Attempting authentication for {connection_id}")
auth_result = await connector.authenticate()
logger.debug(f"Authentication result for {connection_id}: {auth_result}")
-
+
if auth_result:
self.active_connectors[connection_id] = connector
# ... rest of the method
@@ -461,9 +483,7 @@ class ConnectionManager:
},
}
- def _has_saved_credentials_for_user(
- self, connector_type: str, user_id: Optional[str]
- ) -> bool:
+ def _has_saved_credentials_for_user(self, connector_type: str, user_id: Optional[str]) -> bool:
"""Check if user has an active saved connection with usable credentials."""
for connection in self.connections.values():
if connection.connector_type != connector_type or not connection.is_active:
@@ -479,9 +499,7 @@ class ConnectionManager:
continue
return False
- def _is_connector_available(
- self, connector_type: str, user_id: Optional[str] = None
- ) -> bool:
+ def _is_connector_available(self, connector_type: str, user_id: Optional[str] = None) -> bool:
"""Check whether connector is available for use by the given user."""
try:
temp_config = ConnectionConfig(
@@ -549,14 +567,12 @@ class ConnectionManager:
return True
return False
-
+
async def get_connection(self, connection_id: str) -> Optional[ConnectionConfig]:
"""Get connection configuration"""
return self.connections.get(connection_id)
- async def get_connection_by_webhook_id(
- self, webhook_id: str
- ) -> Optional[ConnectionConfig]:
+ async def get_connection_by_webhook_id(self, webhook_id: str) -> Optional[ConnectionConfig]:
"""Find a connection by its webhook/subscription ID"""
for connection in self.connections.values():
# Check if the webhook ID is stored in the connection config
@@ -575,12 +591,10 @@ class ConnectionManager:
):
"""Setup webhook subscription if not already configured"""
# Check if webhook is already set up
- if connection_config.config.get(
- "webhook_channel_id"
- ) or connection_config.config.get("subscription_id"):
- logger.info(
- "Webhook subscription already exists", connection_id=connection_id
- )
+ if connection_config.config.get("webhook_channel_id") or connection_config.config.get(
+ "subscription_id"
+ ):
+ logger.info("Webhook subscription already exists", connection_id=connection_id)
return
# Check if webhook URL is configured
@@ -598,9 +612,7 @@ class ConnectionManager:
# Store the subscription and resource IDs in connection config
connection_config.config["webhook_channel_id"] = subscription_id
- connection_config.config["subscription_id"] = (
- subscription_id # Alternative field
- )
+ connection_config.config["subscription_id"] = subscription_id # Alternative field
if getattr(connector, "webhook_resource_id", None):
connection_config.config["resource_id"] = connector.webhook_resource_id
diff --git a/src/services/file_service.py b/src/services/file_service.py
new file mode 100644
index 000000000..d94b25160
--- /dev/null
+++ b/src/services/file_service.py
@@ -0,0 +1,237 @@
+"""
+Service for file-level listing and search via OpenSearch aggregations.
+
+Aggregates document chunks by filename to produce file-level views,
+with support for pagination, filtering, sorting, and fuzzy search.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from config.settings import get_index_name
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+class FileService:
+ """Provides file-level views over the chunk-based OpenSearch index."""
+
+ def __init__(self, session_manager=None):
+ self.session_manager = session_manager
+
+ async def list_files(
+ self,
+ user_id: str,
+ jwt_token: str = None,
+ page: int = 1,
+ page_size: int = 25,
+ sort_by: str = "filename",
+ sort_order: str = "asc",
+ connector_type: Optional[str] = None,
+ mimetype: Optional[str] = None,
+ owner: Optional[str] = None,
+ search: Optional[str] = None,
+ ) -> Dict[str, Any]:
+ """
+ List ingested files with server-side pagination, filtering, and sorting.
+
+ Aggregates chunks by filename using OpenSearch terms aggregation,
+ then paginates and sorts the resulting file list in-memory.
+ """
+ opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)
+
+ query = self._build_filter_query(user_id, connector_type, mimetype, owner, search)
+ agg_body = self._build_file_aggregation(query)
+
+ try:
+ result = await opensearch_client.search(
+ index=get_index_name(),
+ body=agg_body,
+ )
+ except Exception as e:
+ logger.error("Failed to list files", error=str(e))
+ return {"files": [], "total": 0, "page": page, "page_size": page_size}
+
+ files = self._parse_aggregation_buckets(result)
+ files = self._sort_files(files, sort_by, sort_order)
+
+ total = len(files)
+ start = (page - 1) * page_size
+ end = start + page_size
+ paginated = files[start:end]
+
+ return {
+ "files": paginated,
+ "total": total,
+ "page": page,
+ "page_size": page_size,
+ }
+
+ async def search_files(
+ self,
+ user_id: str,
+ jwt_token: str = None,
+ query: str = "",
+ page: int = 1,
+ page_size: int = 25,
+ connector_type: Optional[str] = None,
+ mimetype: Optional[str] = None,
+ owner: Optional[str] = None,
+ ) -> Dict[str, Any]:
+ """
+ Search files by name with fuzzy/prefix matching.
+
+ Uses wildcard, match_phrase_prefix, and fuzzy queries on the
+ filename field, then aggregates matching chunks into file-level results.
+ """
+ return await self.list_files(
+ user_id=user_id,
+ jwt_token=jwt_token,
+ page=page,
+ page_size=page_size,
+ sort_by="filename",
+ sort_order="asc",
+ connector_type=connector_type,
+ mimetype=mimetype,
+ owner=owner,
+ search=query,
+ )
+
+ def _build_filter_query(
+ self,
+ user_id: str,
+ connector_type: Optional[str] = None,
+ mimetype: Optional[str] = None,
+ owner: Optional[str] = None,
+ search: Optional[str] = None,
+ ) -> Dict[str, Any]:
+ """Build the bool query with optional filters + filename search."""
+ must = []
+ filter_clauses = []
+
+ if connector_type:
+ filter_clauses.append({"term": {"connector_type": connector_type}})
+ if mimetype:
+ filter_clauses.append({"term": {"mimetype": mimetype}})
+ if owner:
+ filter_clauses.append({"term": {"owner": owner}})
+
+ if search:
+ # Combine wildcard (partial), prefix, and fuzzy for flexible matching
+ must.append(
+ {
+ "bool": {
+ "should": [
+ {"wildcard": {"filename": {"value": f"*{search.lower()}*"}}},
+ {"prefix": {"filename": search.lower()}},
+ ],
+ "minimum_should_match": 1,
+ }
+ }
+ )
+
+ query = {"bool": {"filter": filter_clauses}}
+ if must:
+ query["bool"]["must"] = must
+
+ return query
+
+ def _build_file_aggregation(self, query: Dict[str, Any]) -> Dict[str, Any]:
+ """Build the OpenSearch aggregation body for file-level grouping."""
+ return {
+ "size": 0,
+ "query": query,
+ "aggs": {
+ "files": {
+ "terms": {
+ "field": "filename",
+ "size": 10000,
+ },
+ "aggs": {
+ "file_metadata": {
+ "top_hits": {
+ "size": 1,
+ "_source": [
+ "document_id",
+ "filename",
+ "mimetype",
+ "file_size",
+ "source_url",
+ "owner",
+ "owner_name",
+ "owner_email",
+ "connector_type",
+ "embedding_model",
+ "embedding_dimensions",
+ "indexed_time",
+ "allowed_users",
+ "allowed_groups",
+ ],
+ "sort": [{"indexed_time": {"order": "desc"}}],
+ }
+ },
+ "chunk_count": {"value_count": {"field": "_id"}},
+ },
+ }
+ },
+ }
+
+ def _parse_aggregation_buckets(self, result: Dict[str, Any]) -> List[Dict[str, Any]]:
+ """Parse OpenSearch aggregation buckets into file dicts."""
+ buckets = result.get("aggregations", {}).get("files", {}).get("buckets", [])
+
+ files = []
+ for bucket in buckets:
+ hits = bucket.get("file_metadata", {}).get("hits", {}).get("hits", [])
+ if not hits:
+ continue
+
+ source = hits[0].get("_source", {})
+ files.append(
+ {
+ "filename": bucket["key"],
+ "document_id": source.get("document_id", ""),
+ "mimetype": source.get("mimetype", ""),
+ "file_size": source.get("file_size", 0),
+ "source_url": source.get("source_url", ""),
+ "owner": source.get("owner", ""),
+ "owner_name": source.get("owner_name", ""),
+ "owner_email": source.get("owner_email", ""),
+ "connector_type": source.get("connector_type", ""),
+ "embedding_model": source.get("embedding_model", ""),
+ "embedding_dimensions": source.get("embedding_dimensions"),
+ "indexed_time": source.get("indexed_time", ""),
+ "chunk_count": bucket.get("chunk_count", {}).get("value", 0),
+ "allowed_users": source.get("allowed_users", []),
+ "allowed_groups": source.get("allowed_groups", []),
+ }
+ )
+
+ return files
+
+ def _sort_files(
+ self,
+ files: List[Dict[str, Any]],
+ sort_by: str,
+ sort_order: str,
+ ) -> List[Dict[str, Any]]:
+ """Sort file list by the given field."""
+ valid_sort_fields = {
+ "filename",
+ "file_size",
+ "mimetype",
+ "indexed_time",
+ "connector_type",
+ "chunk_count",
+ "owner",
+ }
+ if sort_by not in valid_sort_fields:
+ sort_by = "filename"
+
+ reverse = sort_order.lower() == "desc"
+
+ return sorted(
+ files,
+ key=lambda f: f.get(sort_by) or "",
+ reverse=reverse,
+ )
diff --git a/src/services/search_service.py b/src/services/search_service.py
index 0a55eb01f..b4a0f8935 100644
--- a/src/services/search_service.py
+++ b/src/services/search_service.py
@@ -118,9 +118,7 @@ class SearchService:
available_models = []
failed_models: list = []
- opensearch_client = self.session_manager.get_user_opensearch_client(
- user_id, jwt_token
- )
+ opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)
if not is_wildcard_match_all:
# Build filter clauses first so we can use them in model detection
@@ -141,9 +139,7 @@ class SearchService:
if len(values) == 0:
# Empty array means "match nothing" - use impossible filter
- filter_clauses.append(
- {"term": {field_name: "__IMPOSSIBLE_VALUE__"}}
- )
+ filter_clauses.append({"term": {field_name: "__IMPOSSIBLE_VALUE__"}})
elif len(values) == 1:
# Single value filter
filter_clauses.append({"term": {field_name: values[0]}})
@@ -156,27 +152,22 @@ class SearchService:
agg_query = {
"size": 0,
"aggs": {
- "embedding_models": {
- "terms": {
- "field": "embedding_model",
- "size": 10
- }
- }
- }
+ "embedding_models": {"terms": {"field": "embedding_model", "size": 10}}
+ },
}
# Apply filters to model detection if any exist
if filter_clauses:
- agg_query["query"] = {
- "bool": {
- "filter": filter_clauses
- }
- }
+ agg_query["query"] = {"bool": {"filter": filter_clauses}}
agg_result = await opensearch_client.search(
index=get_index_name(), body=agg_query, params={"terminate_after": 0}
)
- buckets = agg_result.get("aggregations", {}).get("embedding_models", {}).get("buckets", [])
+ buckets = (
+ agg_result.get("aggregations", {})
+ .get("embedding_models", {})
+ .get("buckets", [])
+ )
available_models = [b["key"] for b in buckets if b["key"]]
if not available_models:
@@ -187,10 +178,12 @@ class SearchService:
"Detected embedding models in corpus",
available_models=available_models,
model_counts={b["key"]: b["doc_count"] for b in buckets},
- with_filters=len(filter_clauses) > 0
+ with_filters=len(filter_clauses) > 0,
)
except Exception as e:
- logger.warning("Failed to detect embedding models, using configured model", error=str(e))
+ logger.warning(
+ "Failed to detect embedding models, using configured model", error=str(e)
+ )
available_models = [embedding_model]
# Parallelize embedding generation for all models
@@ -219,9 +212,9 @@ class SearchService:
model=formatted_model, input=[query]
)
# Try to get embedding - some providers return .embedding, others return ['embedding']
- embedding = getattr(resp.data[0], 'embedding', None)
+ embedding = getattr(resp.data[0], "embedding", None)
if embedding is None:
- embedding = resp.data[0]['embedding']
+ embedding = resp.data[0]["embedding"]
return model_name, embedding
except Exception as e:
last_exception = e
@@ -232,9 +225,7 @@ class SearchService:
attempts=attempts,
error=str(e),
)
- raise RuntimeError(
- f"Failed to embed with model {model_name}"
- ) from e
+ raise RuntimeError(f"Failed to embed with model {model_name}") from e
logger.warning(
"Retrying embedding generation",
@@ -247,9 +238,7 @@ class SearchService:
delay = min(delay * 2, EMBED_RETRY_MAX_DELAY)
# Should not reach here, but guard in case
- raise RuntimeError(
- f"Failed to embed with model {model_name}"
- ) from last_exception
+ raise RuntimeError(f"Failed to embed with model {model_name}") from last_exception
# Run all embeddings in parallel, tolerating per-model failures so
# one broken model (e.g. provider credentials removed after ingest)
@@ -298,9 +287,7 @@ class SearchService:
if len(values) == 0:
# Empty array means "match nothing" - use impossible filter
- filter_clauses.append(
- {"term": {field_name: "__IMPOSSIBLE_VALUE__"}}
- )
+ filter_clauses.append({"term": {field_name: "__IMPOSSIBLE_VALUE__"}})
elif len(values) == 1:
# Single value filter
filter_clauses.append({"term": {field_name: values[0]}})
@@ -324,15 +311,17 @@ class SearchService:
for model_name, embedding_vector in query_embeddings.items():
field_name = get_embedding_field_name(model_name)
embedding_fields_to_check.append(field_name)
- knn_queries.append({
- "knn": {
- field_name: {
- "vector": embedding_vector,
- "k": 50,
- "num_candidates": 1000,
+ knn_queries.append(
+ {
+ "knn": {
+ field_name: {
+ "vector": embedding_vector,
+ "k": 50,
+ "num_candidates": 1000,
+ }
}
}
- })
+ )
# Only require an embedding field when we actually have embeddings
# to match against — otherwise we'd filter out every doc in keyword
@@ -345,12 +334,14 @@ class SearchService:
# Allow them through by matching on their embedding_model value.
if failed_models:
exists_should.append({"terms": {"embedding_model": failed_models}})
- all_filters.append({
- "bool": {
- "should": exists_should,
- "minimum_should_match": 1,
+ all_filters.append(
+ {
+ "bool": {
+ "should": exists_should,
+ "minimum_should_match": 1,
+ }
}
- })
+ )
logger.debug(
"Building hybrid query with filters",
@@ -365,39 +356,43 @@ class SearchService:
# the multi_match boost so keyword scoring isn't artificially damped.
should_clauses = []
if knn_queries:
- should_clauses.append({
- "dis_max": {
- "tie_breaker": 0.0, # Take only the best match, no blending
- "boost": 0.7, # 70% weight for semantic search
- "queries": knn_queries,
- }
- })
- should_clauses.extend([
- {
- "multi_match": {
- "query": query,
- "fields": ["text^2", "filename^1.5"],
- "type": "best_fields",
- "operator": "or",
- "fuzziness": "AUTO:4,7",
- "boost": 0.3 if knn_queries else 1.0,
+ should_clauses.append(
+ {
+ "dis_max": {
+ "tie_breaker": 0.0, # Take only the best match, no blending
+ "boost": 0.7, # 70% weight for semantic search
+ "queries": knn_queries,
+ }
}
- },
- {
- # Prefix fallback for partial input (e.g. "vita" -> "vitamin").
- # Avoid bool_prefix here because our current mappings are:
- # - text: standard "text" (not search_as_you_type / edge-ngram)
- # - filename: "keyword"
- # match_phrase_prefix with a bounded expansion is safer.
- "match_phrase_prefix": {
- "text": {
+ )
+ should_clauses.extend(
+ [
+ {
+ "multi_match": {
"query": query,
- "max_expansions": 50,
- "boost": 0.25,
+ "fields": ["text^2", "filename^1.5"],
+ "type": "best_fields",
+ "operator": "or",
+ "fuzziness": "AUTO:4,7",
+ "boost": 0.3 if knn_queries else 1.0,
}
- }
- },
- ])
+ },
+ {
+ # Prefix fallback for partial input (e.g. "vita" -> "vitamin").
+ # Avoid bool_prefix here because our current mappings are:
+ # - text: standard "text" (not search_as_you_type / edge-ngram)
+ # - filename: "keyword"
+ # match_phrase_prefix with a bounded expansion is safer.
+ "match_phrase_prefix": {
+ "text": {
+ "query": query,
+ "max_expansions": 50,
+ "boost": 0.25,
+ }
+ }
+ },
+ ]
+ )
query_block = {
"bool": {
@@ -445,9 +440,9 @@ class SearchService:
if not is_wildcard_match_all and query_embeddings:
try:
fallback_search_body = copy.deepcopy(search_body)
- knn_query_blocks = (
- fallback_search_body["query"]["bool"]["should"][0]["dis_max"]["queries"]
- )
+ knn_query_blocks = fallback_search_body["query"]["bool"]["should"][0]["dis_max"][
+ "queries"
+ ]
for query_candidate in knn_query_blocks:
knn_section = query_candidate.get("knn")
if isinstance(knn_section, dict):
@@ -457,7 +452,7 @@ class SearchService:
except (KeyError, IndexError, AttributeError, TypeError):
fallback_search_body = None
- # Authentication required - DLS will handle document filtering automatically
+ # Authentication required - ACL filter is applied at the application layer above
logger.debug(
"search_service authentication info",
user_id=user_id,
@@ -468,12 +463,14 @@ class SearchService:
return {"results": [], "error": "Authentication required"}
# Get user's OpenSearch client with JWT for OIDC auth through session manager
- opensearch_client = self.session_manager.get_user_opensearch_client(
- user_id, jwt_token
- )
+ opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)
from opensearchpy.exceptions import RequestError
- from utils.opensearch_utils import OpenSearchDiskSpaceError, is_disk_space_error, DISK_SPACE_ERROR_MESSAGE
+ from utils.opensearch_utils import (
+ OpenSearchDiskSpaceError,
+ is_disk_space_error,
+ DISK_SPACE_ERROR_MESSAGE,
+ )
search_params = {"terminate_after": 0}
@@ -531,9 +528,7 @@ class SearchService:
error=str(e),
)
raise OpenSearchDiskSpaceError(DISK_SPACE_ERROR_MESSAGE) from e
- logger.error(
- "OpenSearch query failed", error=str(e), search_body=search_body
- )
+ logger.error("OpenSearch query failed", error=str(e), search_body=search_body)
# Re-raise the exception so the API returns the error to frontend
raise
@@ -566,11 +561,7 @@ class SearchService:
# to avoid broad semantic spillover for unique lookups.
normalized_query = query.strip().lower()
aggregations = results.get("aggregations", {})
- if (
- normalized_query
- and not is_wildcard_match_all
- and len(normalized_query) >= 4
- ):
+ if normalized_query and not is_wildcard_match_all and len(normalized_query) >= 4:
exact_files = {
filename
for chunk in chunks
@@ -598,8 +589,7 @@ class SearchService:
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
- {"key": key, "doc_count": count}
- for key, count in counts.most_common()
+ {"key": key, "doc_count": count} for key, count in counts.most_common()
],
}
@@ -673,4 +663,4 @@ class SearchService:
set_search_limit(limit)
set_score_threshold(score_threshold)
- return await self.search_tool(query, embedding_model=embedding_model)
\ No newline at end of file
+ return await self.search_tool(query, embedding_model=embedding_model)
diff --git a/tests/unit/api/test_acl.py b/tests/unit/api/test_acl.py
new file mode 100644
index 000000000..72383c0a2
--- /dev/null
+++ b/tests/unit/api/test_acl.py
@@ -0,0 +1,68 @@
+import json
+import sys
+from pathlib import Path
+from unittest.mock import AsyncMock
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+
+class FakeSessionManager:
+ def __init__(self, opensearch_client):
+ self.opensearch_client = opensearch_client
+
+ def get_user_opensearch_client(self, user_id, jwt_token):
+ return self.opensearch_client
+
+
+class FakeUser:
+ user_id = "reader"
+ jwt_token = "token"
+
+
+async def test_get_document_acl_returns_read_only_acl_for_document():
+ from api import acl as acl_module
+
+ opensearch_client = AsyncMock()
+ opensearch_client.search = AsyncMock(
+ return_value={
+ "hits": {
+ "hits": [
+ {
+ "_source": {
+ "owner": "owner",
+ "allowed_users": ["reader"],
+ "allowed_groups": ["engineering"],
+ }
+ }
+ ]
+ }
+ }
+ )
+
+ response = await acl_module.get_document_acl(
+ "roadmap.pdf",
+ session_manager=FakeSessionManager(opensearch_client),
+ user=FakeUser(),
+ )
+
+ assert response.status_code == 200
+ assert json.loads(response.body) == {
+ "owner": "owner",
+ "allowed_users": ["reader"],
+ "allowed_groups": ["engineering"],
+ }
+ search_body = opensearch_client.search.await_args.kwargs["body"]
+ assert search_body["_source"] == ["owner", "allowed_users", "allowed_groups"]
+
+
+def test_document_share_methods_are_not_exposed():
+ from api import acl as acl_module
+
+ assert not hasattr(acl_module, "share_document")
+ assert not hasattr(acl_module, "unshare_document")
diff --git a/tests/unit/test_query_acl_filtering.py b/tests/unit/test_query_acl_filtering.py
new file mode 100644
index 000000000..4917996ff
--- /dev/null
+++ b/tests/unit/test_query_acl_filtering.py
@@ -0,0 +1,45 @@
+from pathlib import Path
+
+from src.services.file_service import FileService
+
+
+def test_file_service_query_omits_application_acl_filter():
+ query = FileService()._build_filter_query(
+ user_id="user-123",
+ connector_type="google_drive",
+ mimetype="application/pdf",
+ search="roadmap",
+ )
+
+ filters = query["bool"]["filter"]
+
+ assert filters == [
+ {"term": {"connector_type": "google_drive"}},
+ {"term": {"mimetype": "application/pdf"}},
+ {"term": {"owner": "[email protected]"}},
+ ]
+ assert query["bool"]["must"] == [
+ {
+ "bool": {
+ "should": [
+ {"wildcard": {"filename": {"value": "*roadmap*"}}},
+ {"prefix": {"filename": "roadmap"}},
+ ],
+ "minimum_should_match": 1,
+ }
+ }
+ ]
+
+
+def test_service_query_paths_do_not_apply_document_visibility_filters():
+ repo_root = Path(__file__).resolve().parents[2]
+ helper_name = "build" + "_acl_filter"
+ query_path_files = [
+ repo_root / "src/services/file_service.py",
+ repo_root / "src/services/search_service.py",
+ ]
+
+ for source_file in query_path_files:
+ source = source_file.read_text()
+ assert helper_name not in source