OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat: Knowledge table optimization (#2202) (#2203)
Jason Dongcb9244b54fe5
diff --git a/frontend/app/api/queries/useListFiles.ts b/frontend/app/api/queries/useListFiles.tsindex b5b06f385..7a28571c6 100644--- a/frontend/app/api/queries/useListFiles.ts+++ b/frontend/app/api/queries/useListFiles.ts@@ -14,13 +14,16 @@ export interface ListFilesParams {mimetype?: string;owner?: string;search?: string;+ afterKey?: Record<string, unknown> | null; //composite pagination cursor, could be undefined in the beginning}export interface ListFilesResponse {files: File[];total: number;+ is_approximate: boolean;page: number;page_size: number;+ after_key: Record<string, unknown> | null;}export const useListFiles = (@@ -41,8 +44,11 @@ export const useListFiles = (if (params.mimetype) searchParams.set("mimetype", params.mimetype);if (params.owner) searchParams.set("owner", params.owner);if (params.search) searchParams.set("search", params.search);+ if (params.afterKey)+ searchParams.set("after_key", JSON.stringify(params.afterKey));++ const url = `/api/v2/files?${searchParams.toString()}`; //v2 endpoint- const url = `/api/files?${searchParams.toString()}`;const response = await fetch(url);if (!response.ok) {@@ -56,7 +62,6 @@ export const useListFiles = (const data = await response.json();- // Map server response to File interfaceconst files: File[] = (data.files || []).map((f: Record<string, unknown>) => ({filename: (f.filename as string) || "",@@ -76,12 +81,16 @@ export const useListFiles = (}),);- return {+ const result: ListFilesResponse = {files,total: data.total || 0,+ is_approximate: data.is_approximate ?? true,page: data.page || 1,page_size: data.page_size || 25,+ after_key: data.after_key ?? null,};++ return result;}return useQuery(diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsxindex a11408682..648f456cd 100644--- a/frontend/app/knowledge/page.tsx+++ b/frontend/app/knowledge/page.tsx@@ -4,6 +4,7 @@ import { useQueryClient } from "@tanstack/react-query";import {type CheckboxSelectionCallbackParams,type ColDef,+ type ColumnState,type GetRowIdParams,themeQuartz,type ValueFormatterParams,@@ -32,6 +33,7 @@ import "@/components/AgGrid/agGridStyles.css";import { toast } from "sonner";import { KnowledgeActionsDropdown } from "@/components/knowledge-actions-dropdown";import { KnowledgeBatchActionsBar } from "@/components/knowledge-batch-actions-bar";+import { KnowledgePaginationFooter } from "@/components/knowledge-pagination-footer";import { KnowledgeSearchBar } from "@/components/knowledge-search-bar";import { KnowledgeSearchInput } from "@/components/knowledge-search-input";import { RequirePermission } from "@/components/require-permission";@@ -128,6 +130,17 @@ function getSourceIcon(connectorType?: string) {}}+const AG_FIELD_TO_SORT_BY: Record<string, string> = {+ filename: "filename",+ size: "file_size",+ mimetype: "mimetype",+ owner: "owner",+ chunkCount: "chunk_count",+ embedding_model: "embedding_model",+ embedding_dimensions: "embedding_dimensions",+ status: "status",+};+function SearchPage() {const isCloudBrand = useIsCloudBrand();const queryClient = useQueryClient();@@ -148,8 +161,9 @@ function SearchPage() {} = useKnowledgeFilter();const [selectedRows, setSelectedRows] = useState<File[]>([]);- // Keep the filter context aware of checked rows so "Create New Filter"- // can pre-populate its sources from the current selection.+ const [sortBy, setSortBy] = useState<string>("filename");+ const [sortOrder, setSortOrder] = useState<"asc" | "desc">("asc");+useEffect(() => {setSelectedSources(selectedRows.flatMap((row) => (row.filename ? [row.filename] : [])),@@ -170,6 +184,16 @@ function SearchPage() {null,);+ const [currentPage, setCurrentPage] = useState(1);+ const [currentPageSize, setCurrentPageSize] = useState(25);++ const cursorCacheRef = useRef<Map<number, Record<string, unknown>>>(+ null as any,+ );+ if (!cursorCacheRef.current) {+ cursorCacheRef.current = new Map();+ }+const handleOpenSyncDialog = useCallback(async () => {setSyncPreview(null);setSyncDialogOpen(true);@@ -269,8 +293,6 @@ function SearchPage() {[taskFiles, tasks],);- // Auto-open unified task panel only when a NEW task file transitions to failed- // (skip initial failed files that already existed on page load).useEffect(() => {const failedFiles = taskFiles.filter((file) => file.status === "failed");const seenKeys = seenFailedFileKeysRef.current;@@ -311,8 +333,6 @@ function SearchPage() {getFailedFileKey,]);- // Use server-side file listing for default/wildcard view; search otherwise.- // Wildcard follows bar text or saved filter query (bar is cleared when a filter is picked).const effectiveSearchText =queryOverride.trim() || parsedFilterData?.query?.trim() || "";const hasActiveFilters = parsedFilterData?.filters@@ -325,12 +345,16 @@ function SearchPage() {const {data: listFilesData,isLoading: isListFilesLoading,+ isFetching: isListFilesFetching,error: listFilesError,isError: isListFilesError,} = useListFiles({- pageSize: 100,- search: isWildcardQuery ? undefined : queryOverride,+ page: currentPage,+ pageSize: currentPageSize,+ sortBy,+ sortOrder,+ afterKey: cursorCacheRef.current.get(currentPage) ?? null,connectorType: listFilesFilterParam(parsedFilterData?.filters?.connector_types,),@@ -355,14 +379,19 @@ function SearchPage() {const { files: searchFiles, warnings: searchWarnings } =searchData as SearchResult;- // Merge data from whichever source is active- const effectiveData: File[] = isWildcardQuery- ? (listFilesData?.files ?? [])- : searchFiles;const isLoading = isWildcardQuery ? isListFilesLoading : isSearchLoading;++ const isFetching = isWildcardQuery ? isListFilesFetching : isSearchLoading;const error = isWildcardQuery ? listFilesError : searchError;const isError = isWildcardQuery ? isListFilesError : isSearchError;+ const effectiveData: File[] = isWildcardQuery+ ? (listFilesData?.files ?? [])+ : searchFiles.slice(+ (currentPage - 1) * currentPageSize,+ currentPage * currentPageSize,+ );+const isOpenragDocsRow = useCallback((file?: File) => {return (file?.connector_type === "openrag_docs" ||@@ -393,26 +422,6 @@ function SearchPage() {return file?.owner_name?.trim() || file?.owner_email?.trim() || "—";}, []);- const normalizeSourceForSort = useCallback((value?: string): string => {- const trimmed = (value || "").trim();- if (!trimmed) {- return "";- }-- try {- const parsed = new URL(trimmed);- const hostname = parsed.hostname.toLowerCase();- const pathname = parsed.pathname.replace(/\/+$/, "").toLowerCase();- return `${hostname}${pathname}`;- } catch {- return trimmed- .toLowerCase()- .replace(/^https?:\/\//, "")- .split(/[?#]/)[0]- .replace(/\/+$/, "");- }- }, []);-const getStatusSortRank = useCallback((status?: File["status"]): number => {switch (status) {case "active":@@ -469,14 +478,30 @@ function SearchPage() {lastErrorRef.current = null;}}, [isError, error]);- // Third arg: saved filter only — draft `parsedFilterData` (create mode) must still show task rows.const fileResults = buildKnowledgeTableRows(effectiveData,taskFiles,Boolean(selectedFilter),);- const gridRows = fileResults;+ const serverTotal = isWildcardQuery+ ? (listFilesData?.total ?? 0)+ : searchFiles.length;+ const gridRows: File[] = fileResults;+ const totalPages = Math.max(1, Math.ceil(serverTotal / currentPageSize));++ useEffect(() => {+ cursorCacheRef.current = new Map();+ setCurrentPage(1);+ }, [effectiveSearchText]);++ // when the server responds with an after_key for page N, cache it as the cursor for page N+1+ useEffect(() => {+ if (listFilesData?.after_key && listFilesData.page) {+ const nextPage = listFilesData.page + 1;+ cursorCacheRef.current.set(nextPage, listFilesData.after_key);+ }+ }, [listFilesData]);const gridRef = useRef<AgGridReact>(null);const gridReadyRef = useRef(false);@@ -493,7 +518,27 @@ function SearchPage() {return gridRef.current?.api ?? null;}, []);- // Re-run only when row identity/status changes, not on every list poll reference.+ const onSortChanged = useCallback(() => {+ const api = getGridApi();+ if (!api) return;++ const sortedCol: ColumnState | undefined = api+ .getColumnState()+ .find((col) => col.sort != null);++ const newSortBy = sortedCol+ ? (AG_FIELD_TO_SORT_BY[sortedCol.colId] ?? sortedCol.colId)+ : "filename";+ const newSortOrder: "asc" | "desc" =+ sortedCol?.sort === "desc" ? "desc" : "asc";++ // Changing sort invalidates all cursors; reset to page 1+ cursorCacheRef.current = new Map();+ setCurrentPage(1);+ setSortBy(newSortBy);+ setSortOrder(newSortOrder);+ }, [getGridApi]);+const gridRowsSelectionKey = useMemo(() =>gridRows@@ -521,19 +566,7 @@ function SearchPage() {field: "filename",headerName: "Source",sortable: true,- comparator: (valueA?: string, valueB?: string) => {- const sourceA = normalizeSourceForSort(valueA);- const sourceB = normalizeSourceForSort(valueB);- if (sourceA === sourceB) {- const fallbackA = (valueA || "").trim().toLowerCase();- const fallbackB = (valueB || "").trim().toLowerCase();- if (fallbackA === fallbackB) {- return 0;- }- return fallbackA < fallbackB ? -1 : 1;- }- return sourceA < sourceB ? -1 : 1;- },+ comparator: () => 0,checkboxSelection: (params: CheckboxSelectionCallbackParams<File>) =>isDeletableKnowledgeRow(params?.data),headerCheckboxSelection: true,@@ -599,8 +632,7 @@ function SearchPage() {headerName: "Size",...(isCloudBrand ? { flex: 1, minWidth: 110 } : {}),sortable: true,- comparator: (valueA?: number, valueB?: number) =>- (valueA || 0) - (valueB || 0),+ comparator: () => 0,valueFormatter: (params: ValueFormatterParams<File>) =>params.value ? formatFileSize(params.value) : "-",cellClass: isCloudBrand ? "text-muted-foreground" : undefined,@@ -622,18 +654,14 @@ function SearchPage() {sortable: true,valueGetter: (params: ValueGetterParams<File>) =>getOwnerLabel(params.data),- comparator: (valueA?: string, valueB?: string) =>- (valueA || "—").localeCompare(valueB || "—", undefined, {- sensitivity: "base",- }),+ comparator: () => 0,},{field: "chunkCount",headerName: "Chunks",...(isCloudBrand ? { flex: 0.9, minWidth: 95 } : {}),sortable: true,- comparator: (valueA?: number, valueB?: number) =>- (valueA || 0) - (valueB || 0),+ comparator: () => 0,valueFormatter: (params: ValueFormatterParams<File>) =>params.data?.chunkCount?.toString() || "-",cellClass: isCloudBrand ? "text-muted-foreground" : undefined,@@ -677,8 +705,7 @@ function SearchPage() {headerName: "Dimensions",...(isCloudBrand ? { flex: 0.9, minWidth: 110 } : { width: 110 }),sortable: true,- comparator: (valueA?: number, valueB?: number) =>- (valueA || 0) - (valueB || 0),+ comparator: () => 0,cellRenderer: ({ data }: CustomCellRendererProps<File>) => (<span className="text-xs text-muted-foreground">{typeof data?.embedding_dimensions === "number"@@ -879,15 +906,6 @@ function SearchPage() {}};- // enables pagination in the grid- const pagination = true;-- // sets 25 rows per page (default is 100)- const paginationPageSize = 25;-- // allows the user to select the page size from a predefined list of page sizes- const paginationPageSizeSelector = [10, 25, 50, 100];-return (<><div className="flex flex-col h-full">@@ -933,7 +951,7 @@ function SearchPage() {</div>) : (/* Search Input Area */- <div className="flex-1 flex items-center flex-shrink-0 flex-wrap-reverse gap-3 mb-6">+ <div className="flex items-center flex-shrink-0 flex-wrap-reverse gap-3 mb-6"><KnowledgeSearchInput /><Button@@ -1045,74 +1063,85 @@ function SearchPage() {</div>)}{isCloudBrand ? (- <AgGridReact- className="w-full overflow-auto border"- columnDefs={columnDefs as ColDef<File>[]}- defaultColDef={defaultColDef}- loading={isLoading || deleteDocumentMutation.isPending}- ref={gridRef}- theme={themeQuartz.withParams({ browserColorScheme: "inherit" })}- rowData={gridRows}- rowSelection="multiple"- getRowId={(params: GetRowIdParams<File>) =>- getFileIdentity(params.data)- }- isRowSelectable={(params) => isDeletableKnowledgeRow(params.data)}- domLayout="normal"- onGridReady={handleGridReady}- onGridPreDestroyed={handleGridPreDestroyed}- onSelectionChanged={onSelectionChanged}- pagination={pagination}- paginationPageSize={paginationPageSize}- paginationPageSizeSelector={paginationPageSizeSelector}- headerHeight={64}- rowHeight={64}- noRowsOverlayComponent={() => (- <div className="text-center pb-[45px]">- <div className="text-lg text-primary font-semibold">- No knowledge- </div>- <div className="text-sm mt-1 text-muted-foreground">- Add files from local or your preferred cloud.+ <div className="flex-1 min-h-0 overflow-hidden">+ <AgGridReact+ className="w-full h-full border"+ columnDefs={columnDefs as ColDef<File>[]}+ defaultColDef={defaultColDef}+ loading={isLoading || deleteDocumentMutation.isPending}+ ref={gridRef}+ theme={themeQuartz.withParams({ browserColorScheme: "inherit" })}+ rowData={gridRows}+ rowSelection="multiple"+ getRowId={(params: GetRowIdParams<File>) =>+ getFileIdentity(params.data)+ }+ isRowSelectable={(params) => isDeletableKnowledgeRow(params.data)}+ domLayout="normal"+ onGridReady={handleGridReady}+ onGridPreDestroyed={handleGridPreDestroyed}+ onSelectionChanged={onSelectionChanged}+ onSortChanged={onSortChanged}+ headerHeight={64}+ rowHeight={64}+ noRowsOverlayComponent={() => (+ <div className="text-center pb-[45px]">+ <div className="text-lg text-primary font-semibold">+ No knowledge+ </div>+ <div className="text-sm mt-1 text-muted-foreground">+ Add files from local or your preferred cloud.+ </div></div>- </div>- )}- />+ )}+ />+ </div>) : (- <AgGridReact- className="w-full overflow-auto"- columnDefs={columnDefs as ColDef<File>[]}- defaultColDef={defaultColDef}- loading={isLoading || deleteDocumentMutation.isPending}- ref={gridRef}- theme={themeQuartz.withParams({ browserColorScheme: "inherit" })}- rowData={gridRows}- rowSelection="multiple"- rowMultiSelectWithClick={false}- suppressRowClickSelection={true}- getRowId={(params: GetRowIdParams<File>) =>- getFileIdentity(params.data)- }- isRowSelectable={(params) => isDeletableKnowledgeRow(params.data)}- domLayout="normal"- onGridReady={handleGridReady}- onGridPreDestroyed={handleGridPreDestroyed}- onSelectionChanged={onSelectionChanged}- pagination={pagination}- paginationPageSize={paginationPageSize}- paginationPageSizeSelector={paginationPageSizeSelector}- noRowsOverlayComponent={() => (- <div className="text-center pb-[45px]">- <div className="text-lg text-primary font-semibold">- No knowledge- </div>- <div className="text-sm mt-1 text-muted-foreground">- Add files from local or your preferred cloud.+ <div className="flex-1 min-h-0 overflow-hidden">+ <AgGridReact+ className="w-full h-full"+ columnDefs={columnDefs as ColDef<File>[]}+ defaultColDef={defaultColDef}+ loading={isLoading || deleteDocumentMutation.isPending}+ ref={gridRef}+ theme={themeQuartz.withParams({ browserColorScheme: "inherit" })}+ rowData={gridRows}+ rowSelection="multiple"+ rowMultiSelectWithClick={false}+ suppressRowClickSelection={true}+ getRowId={(params: GetRowIdParams<File>) =>+ getFileIdentity(params.data)+ }+ isRowSelectable={(params) => isDeletableKnowledgeRow(params.data)}+ domLayout="normal"+ onGridReady={handleGridReady}+ onGridPreDestroyed={handleGridPreDestroyed}+ onSelectionChanged={onSelectionChanged}+ onSortChanged={onSortChanged}+ noRowsOverlayComponent={() => (+ <div className="text-center pb-[45px]">+ <div className="text-lg text-primary font-semibold">+ No knowledge+ </div>+ <div className="text-sm mt-1 text-muted-foreground">+ Add files from local or your preferred cloud.+ </div></div>- </div>- )}- />+ )}+ />+ </div>)}++ <KnowledgePaginationFooter+ currentPage={currentPage}+ currentPageSize={currentPageSize}+ totalPages={totalPages}+ serverTotal={serverTotal}+ isLoading={isFetching}+ cursorCacheRef={cursorCacheRef}+ setCurrentPage={setCurrentPage}+ setCurrentPageSize={setCurrentPageSize}+ /></div>{/* Bulk Delete Confirmation Dialog */}diff --git a/frontend/components/knowledge-pagination-footer.tsx b/frontend/components/knowledge-pagination-footer.tsxnew file mode 100644index 000000000..5b1593682--- /dev/null+++ b/frontend/components/knowledge-pagination-footer.tsx@@ -0,0 +1,109 @@+import React from "react";++export interface KnowledgePaginationFooterProps {+ currentPage: number;+ currentPageSize: number;+ totalPages: number;+ serverTotal: number;+ isLoading?: boolean;+ cursorCacheRef: React.RefObject<Map<number, Record<string, unknown>>>;+ setCurrentPage: React.Dispatch<React.SetStateAction<number>>;+ setCurrentPageSize: React.Dispatch<React.SetStateAction<number>>;+}++export function KnowledgePaginationFooter({+ currentPage,+ currentPageSize,+ totalPages,+ serverTotal,+ isLoading = false,+ cursorCacheRef,+ setCurrentPage,+ setCurrentPageSize,+}: KnowledgePaginationFooterProps) {+ const atFirst = currentPage <= 1;+ const atLast = currentPage >= totalPages;++ return (+ <div+ className="flex items-center justify-end gap-4 border-t border-border bg-background px-3 text-sm text-muted-foreground"+ style={{ height: "var(--ag-pagination-panel-height, 48px)" }}+ >+ {/* page size */}+ <label className="flex items-center gap-1.5 whitespace-nowrap">+ Page Size:+ <select+ value={currentPageSize}+ onChange={(e) => {+ cursorCacheRef.current = new Map();+ setCurrentPageSize(Number(e.target.value));+ setCurrentPage(1);+ }}+ className="cursor-pointer rounded-none border border-border bg-background px-1 py-0.5 text-sm text-muted-foreground"+ >+ {[10, 25, 50, 100].map((n) => (+ <option key={n} value={n}>+ {n}+ </option>+ ))}+ </select>+ </label>++ {/* row summary */}+ <span className="whitespace-nowrap">+ {serverTotal === 0+ ? `0 of 0`+ : `${(currentPage - 1) * currentPageSize + 1} to ${Math.min(+ currentPage * currentPageSize,+ serverTotal,+ )} of ${serverTotal}`}+ </span>++ {/* nav buttons */}+ <div className="flex items-center">+ <button+ type="button"+ aria-label="Back to first page"+ aria-disabled={atFirst || isLoading}+ disabled={atFirst || isLoading}+ onClick={() => {+ if (!atFirst && !isLoading) {+ cursorCacheRef.current = new Map();+ setCurrentPage(1);+ }+ }}+ className="-mr-2 flex h-10 w-10 items-center justify-center rounded text-lg leading-none disabled:cursor-default disabled:opacity-40"+ >+ <span className="pointer-events-none">«</span>+ </button>+ <button+ type="button"+ aria-label="Previous page"+ aria-disabled={atFirst || isLoading}+ disabled={atFirst || isLoading}+ onClick={() => {+ if (!atFirst && !isLoading) setCurrentPage((p) => p - 1);+ }}+ className="flex h-10 w-10 items-center justify-center rounded text-lg leading-none disabled:cursor-default disabled:opacity-40"+ >+ <span className="pointer-events-none">‹</span>+ </button>+ <span className="whitespace-nowrap px-2">+ Page {currentPage} of {totalPages}+ </span>+ <button+ type="button"+ aria-label="Next page"+ aria-disabled={atLast || isLoading}+ disabled={atLast || isLoading}+ onClick={() => {+ if (!atLast && !isLoading) setCurrentPage((p) => p + 1);+ }}+ className="flex h-10 w-10 items-center justify-center rounded text-lg leading-none disabled:cursor-default disabled:opacity-40"+ >+ <span className="pointer-events-none">›</span>+ </button>+ </div>+ </div>+ );+}diff --git a/src/api/files.py b/src/api/files.pyindex e3e0e9664..889d1e8de 100644--- a/src/api/files.py+++ b/src/api/files.py@@ -1,8 +1,8 @@"""File listing and search API endpoints.-Provides server-side file-level views over the chunk-based OpenSearch index,-replacing the client-side chunk-to-file aggregation pattern.+Provides server-side file-level views over the chunk-based OpenSearch index.+Uses terms aggregation with in-memory sort/slice (v1)."""from fastapi import Depends, Querydiff --git a/src/api/v2/__init__.py b/src/api/v2/__init__.pynew file mode 100644index 000000000..2e04b7cf1--- /dev/null+++ b/src/api/v2/__init__.py@@ -0,0 +1,5 @@+"""+OpenRAG API v2++Composite-aggregation endpoints — cursor-based pagination, O(page_size) per request.+"""diff --git a/src/api/v2/files.py b/src/api/v2/files.pynew file mode 100644index 000000000..b6fc1b1f3--- /dev/null+++ b/src/api/v2/files.py@@ -0,0 +1,119 @@+"""+File listing and search API endpoints — v2 (composite aggregation).++Uses composite aggregation for true O(page_size) server-side pagination.+Cursor-based: pass `after_key` (JSON-encoded) to advance pages.+"""++import json++from fastapi import Depends, HTTPException, Query+from fastapi.responses import JSONResponse++from dependencies import get_current_user, get_file_service_v2+from session_manager import User+from utils.logging_config import get_logger++logger = get_logger(__name__)+++def _parse_after_key(after_key: str | None) -> dict | None:+ """Parse a JSON-encoded composite cursor.++ Returns None when after_key is absent.+ Raises HTTPException 400 when the value is present but is not valid JSON+ or does not decode to a dict (e.g. a bare string or number is invalid).+ """+ if not after_key:+ return None+ try:+ parsed = json.loads(after_key)+ except (json.JSONDecodeError, ValueError) as err:+ raise HTTPException(status_code=400, detail="after_key is not valid JSON") from err+ if not isinstance(parsed, dict):+ raise HTTPException(status_code=400, detail="after_key must be a JSON object")+ return parsed+++async def list_files(+ page: int = Query(+ 1, ge=1, description="Page number (for display only; navigation uses after_key cursor)"+ ),+ page_size: int = Query(25, ge=1, le=500, description="Items per page"),+ sort_by: str = Query("filename", description="Sort field"),+ sort_order: str = Query("asc", pattern="^(asc|desc)$", description="Sort order"),+ connector_type: str | None = Query(None, description="Filter by connector type"),+ mimetype: str | None = Query(None, description="Filter by MIME type"),+ owner: str | None = Query(None, description="Filter by owner"),+ search: str | None = Query(None, description="Search filename"),+ after_key: str | None = Query(None, description="Composite pagination cursor (JSON-encoded)"),+ file_service=Depends(get_file_service_v2),+ user: User = Depends(get_current_user),+):+ """List ingested files with composite-aggregation pagination, filtering, and sorting."""+ parsed_after_key = _parse_after_key(after_key)++ try:+ result = await file_service.list_files(+ user_id=user.user_id,+ jwt_token=user.jwt_token,+ page=page,+ page_size=page_size,+ sort_by=sort_by,+ sort_order=sort_order,+ connector_type=connector_type,+ mimetype=mimetype,+ owner=owner,+ search=search,+ after_key=parsed_after_key,+ )+ return JSONResponse(result)+ except Exception as e:+ logger.error("Failed to list files (v2)", error=str(e))+ from utils.opensearch_utils import AUTH_ERROR_MESSAGE, is_opensearch_auth_error++ if is_opensearch_auth_error(e):+ return JSONResponse({"error": AUTH_ERROR_MESSAGE}, status_code=401)+ return JSONResponse(+ {"error": "Failed to list files"},+ status_code=500,+ )+++async def search_files(+ q: str = Query(..., min_length=1, description="Search query"),+ page: int = Query(1, ge=1, description="Page number"),+ page_size: int = Query(25, ge=1, le=500, description="Items per page"),+ connector_type: str | None = Query(None, description="Filter by connector type"),+ mimetype: str | None = Query(None, description="Filter by MIME type"),+ owner: str | None = Query(None, description="Filter by owner"),+ after_key: str | None = Query(None, description="Composite pagination cursor (JSON-encoded)"),+ file_service=Depends(get_file_service_v2),+ user: User = Depends(get_current_user),+):+ """Search files by name with fuzzy/partial matching (v2 — composite pagination)."""+ parsed_after_key = _parse_after_key(after_key)++ try:+ result = await file_service.search_files(+ user_id=user.user_id,+ jwt_token=user.jwt_token,+ query=q,+ page=page,+ page_size=page_size,+ connector_type=connector_type,+ mimetype=mimetype,+ owner=owner,+ after_key=parsed_after_key,+ )+ return JSONResponse(result)+ except Exception as e:+ logger.error("Failed to search files (v2)", error=str(e))+ from utils.opensearch_utils import AUTH_ERROR_MESSAGE, is_opensearch_auth_error++ if is_opensearch_auth_error(e):+ return JSONResponse({"error": AUTH_ERROR_MESSAGE}, status_code=401)+ return JSONResponse(+ {"error": "Failed to search files"},+ status_code=500,+ )diff --git a/src/app/container.py b/src/app/container.pyindex a7a7918c5..27d538488 100644--- a/src/app/container.py+++ b/src/app/container.py@@ -23,6 +23,7 @@ from services.dls_principal_service import DLSPrincipalServicefrom services.docling_polling_service import DoclingPollingServicefrom services.document_index_writer import DocumentIndexWriterfrom services.document_service import DocumentService+from services.file_service_v2 import FileServiceV2from services.flows_service import FlowsServicefrom services.group_acl_service import GroupACLServicefrom services.ingest_preview_service import IngestPreviewService@@ -213,6 +214,8 @@ async def initialize_services():session_ownership_service._session_factory = _lazy_session_factoryconversation_persistence._session_factory = _lazy_session_factory+ file_service_v2 = FileServiceV2(session_manager=session_manager)+return {"document_service": document_service,"search_service": search_service,@@ -237,4 +240,5 @@ async def initialize_services():"ingest_preview_service": ingest_preview_service,"rbac_service": rbac_service,"workspace_config_service": workspace_config_service,+ "file_service_v2": file_service_v2,}diff --git a/src/app/routes/internal.py b/src/app/routes/internal.pyindex 25e2492b1..c9703e833 100644--- a/src/app/routes/internal.py+++ b/src/app/routes/internal.py@@ -33,6 +33,7 @@ from api import (from api import keys as api_keysfrom api.health import health_check, opensearch_health_readyfrom api.schemas.tasks import ErrorResponse, TaskRetryResponse+from api.v2 import files as files_v2from connectors.registry import get_connector_classes@@ -121,10 +122,14 @@ def register_internal_routes(app: FastAPI):# Search endpointapp.add_api_route("/search", search.search, methods=["POST"], tags=["internal"])- # File listing/search endpoints+ # File listing/search endpoints (v1 — terms-agg, in-memory sort)app.add_api_route("/files", files.list_files, methods=["GET"], tags=["internal"])app.add_api_route("/files/search", files.search_files, methods=["GET"], tags=["internal"])+ # File listing/search endpoints (v2 — composite-agg, cursor pagination)+ app.add_api_route("/v2/files/search", files_v2.search_files, methods=["GET"], tags=["internal"])+ app.add_api_route("/v2/files", files_v2.list_files, methods=["GET"], tags=["internal"])+# Knowledge Filter endpointsapp.add_api_route("/knowledge-filter",diff --git a/src/dependencies.py b/src/dependencies.pyindex 8c4334025..b9fb51654 100644--- a/src/dependencies.py+++ b/src/dependencies.py@@ -66,6 +66,7 @@ __all__ = ["get_api_key_service","get_api_key_user_async","get_auth_service",+ "get_file_service_v2","get_chat_service","get_connector_service","get_current_user",@@ -176,6 +177,10 @@ def get_api_key_service(services: dict = Depends(get_services)):return services["api_key_service"]+def get_file_service_v2(services: dict = Depends(get_services)):+ return services["file_service_v2"]++def get_flows_service(services: dict = Depends(get_services)):return services["flows_service"]diff --git a/src/services/file_service.py b/src/services/file_service.pyindex ce3f86786..dc5f620d7 100644--- a/src/services/file_service.py+++ b/src/services/file_service.py@@ -238,8 +238,9 @@ class FileService:reverse = sort_order.lower() == "desc"- return sorted(- files,- key=lambda f: f.get(sort_by) or "",- reverse=reverse,- )+ _numeric_sort_fields = {"file_size", "chunk_count"}++ def _sort_key(f):+ return f.get(sort_by) or (0 if sort_by in _numeric_sort_fields else "")++ return sorted(files, key=_sort_key, reverse=reverse)diff --git a/src/services/file_service_v2.py b/src/services/file_service_v2.pynew file mode 100644index 000000000..6a4607dd0--- /dev/null+++ b/src/services/file_service_v2.py@@ -0,0 +1,467 @@+"""+File service v2 — composite aggregation pagination.++Uses OpenSearch composite aggregation for O(page_size) server-side+pagination. Only page_size buckets are processed per request regardless of+total file count.++Sort-field notes+----------------+* Fields in _COMPOSITE_SORT_FIELDS are handled entirely by the composite agg —+ ordering is globally correct and cursor-paginated.+* "owner" is a real keyword field so it lives in _COMPOSITE_SORT_FIELDS.+* "chunk_count" is a derived metric (value_count sub-agg). Composite aggs+ cannot order by sub-aggregation metrics, so sorting by chunk_count falls back+ to a classic terms aggregation sorted by the sub-agg, with offset-based+ pagination (_build_terms_aggregation_for_chunk_count). This is O(from+size)+ at the shard level but gives a globally correct sort.+"""++from typing import Any++from config.settings import get_index_name+from utils.logging_config import get_logger++logger = get_logger(__name__)++_COMPOSITE_SORT_FIELDS: dict[str, str] = {+ "filename": "filename",+ "file_size": "file_size",+ "mimetype": "mimetype",+ "indexed_time": "indexed_time",+ "connector_type": "connector_type",+ "embedding_model": "embedding_model",+ "owner": "owner",+}+++class FileServiceV2:+ """File-level views via composite aggregation (v2 — cursor pagination)."""++ def __init__(self, session_manager=None):+ self.session_manager = session_manager++ async def list_files(+ self,+ user_id: str,+ jwt_token: str = None,+ page: int = 1,+ page_size: int = 25,+ sort_by: str = "filename",+ sort_order: str = "asc",+ connector_type: str | None = None,+ mimetype: str | None = None,+ owner: str | None = None,+ search: str | None = None,+ after_key: dict | None = None,+ ) -> dict[str, Any]:+ """+ List files with server-side pagination via composite aggregation.++ Cost is O(page_size) per request (as opposed to returning all unique+ file sizes from OpenSearch). Returns after_key for the next page+ (None when on the last page), plus an approximate total from a+ cardinality aggregation.++ Exception: when sort_by="chunk_count" a terms aggregation is used+ instead (composite aggs cannot order by sub-agg metrics). Pagination+ is offset-based in that path; after_key is always None.+ """+ opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)++ query = self._build_filter_query(user_id, connector_type, mimetype, owner, search)+ total, is_approximate = await self._get_file_count(opensearch_client, query)++ if sort_by == "chunk_count":+ return await self._list_files_by_chunk_count(+ opensearch_client=opensearch_client,+ query=query,+ page=page,+ page_size=page_size,+ sort_order=sort_order,+ total=total,+ is_approximate=is_approximate,+ )++ if page > 1 and after_key is None:+ return {+ "files": [],+ "total": total,+ "is_approximate": is_approximate,+ "page": page,+ "page_size": page_size,+ "after_key": None,+ }++ composite_sort_field = _COMPOSITE_SORT_FIELDS.get(sort_by, "filename")++ agg_body = self._build_composite_aggregation(+ query=query,+ page_size=page_size,+ sort_field=composite_sort_field,+ sort_order=sort_order,+ after_key=after_key,+ )++ try:+ result = await opensearch_client.search(+ index=get_index_name(),+ body=agg_body,+ )+ except Exception as e:+ logger.error("Failed to list files (v2)", error=str(e))+ from utils.opensearch_utils import is_opensearch_auth_error++ if is_opensearch_auth_error(e):+ raise+ return {+ "files": [],+ "total": 0,+ "is_approximate": False,+ "page": page,+ "page_size": page_size,+ "after_key": None,+ }++ raw_bucket_count = len(result.get("aggregations", {}).get("files", {}).get("buckets", []))+ files, next_after_key = self._parse_composite_buckets(result)++ if raw_bucket_count < page_size: # final page, no after_key+ next_after_key = None++ return {+ "files": files,+ "total": total,+ "is_approximate": is_approximate,+ "page": page,+ "page_size": page_size,+ "after_key": next_after_key,+ }++ async def _list_files_by_chunk_count(+ self,+ opensearch_client: Any,+ query: dict[str, Any],+ page: int,+ page_size: int,+ sort_order: str,+ total: int,+ is_approximate: bool,+ ) -> dict[str, Any]:+ """+ List files sorted globally by chunk_count using a terms aggregation.++ Terms aggs support ordering by sub-aggregation metrics, which composite+ aggs do not. Pagination is offset-based (from = (page-1)*page_size).+ """+ offset = (page - 1) * page_size+ agg_body = self._build_terms_aggregation_for_chunk_count(+ query=query,+ page_size=page_size,+ offset=offset,+ sort_order=sort_order,+ )++ try:+ result = await opensearch_client.search(+ index=get_index_name(),+ body=agg_body,+ )+ except Exception as e:+ logger.error("Failed to list files by chunk_count (v2)", error=str(e))+ from utils.opensearch_utils import is_opensearch_auth_error++ if is_opensearch_auth_error(e):+ raise+ return {+ "files": [],+ "total": 0,+ "page": page,+ "page_size": page_size,+ "after_key": None,+ }++ files = self._parse_terms_buckets(result, offset=offset)++ return {+ "files": files,+ "total": total,+ "is_approximate": is_approximate,+ "page": page,+ "page_size": page_size,+ "after_key": None, # offset pagination; no cursor+ }++ async def search_files(+ self,+ user_id: str,+ jwt_token: str = None,+ query: str = "",+ page: int = 1,+ page_size: int = 25,+ connector_type: str | None = None,+ mimetype: str | None = None,+ owner: str | None = None,+ after_key: dict | None = None,+ ) -> dict[str, Any]:+ """Search files by name with fuzzy/prefix matching."""+ return await self.list_files(+ user_id=user_id,+ jwt_token=jwt_token,+ page=page,+ page_size=page_size,+ sort_by="filename",+ sort_order="asc",+ connector_type=connector_type,+ mimetype=mimetype,+ owner=owner,+ search=query,+ after_key=after_key,+ )++ def _build_filter_query(+ self,+ user_id: str,+ connector_type: str | None = None,+ mimetype: str | None = None,+ owner: str | None = None,+ search: str | None = None,+ ) -> dict[str, Any]:+ must = []+ filter_clauses = []++ if connector_type:+ filter_clauses.append({"term": {"connector_type": connector_type}})+ if mimetype:+ filter_clauses.append({"term": {"mimetype": mimetype}})+ if owner:+ filter_clauses.append({"term": {"owner": owner}})++ if search:+ must.append(+ {+ "bool": {+ "should": [+ {"wildcard": {"filename": {"value": f"*{search.lower()}*"}}},+ {"prefix": {"filename": search.lower()}},+ ],+ "minimum_should_match": 1,+ }+ }+ )++ query: dict[str, Any] = {"bool": {"filter": filter_clauses}}+ if must:+ query["bool"]["must"] = must+ return query++ def _build_composite_aggregation(+ self,+ query: dict[str, Any],+ page_size: int,+ sort_field: str,+ sort_order: str,+ after_key: dict | None,+ ) -> dict[str, Any]:+ composite: dict[str, Any] = {+ "size": page_size,+ "sources": [+ {+ sort_field: {+ "terms": {+ "field": sort_field,+ "order": sort_order,+ "missing_bucket": True,+ }+ }+ },+ *(+ [+ {+ "filename_tiebreak": {+ "terms": {+ "field": "filename",+ "order": sort_order,+ "missing_bucket": True,+ }+ }+ }+ ]+ if sort_field != "filename"+ else []+ ),+ ],+ }++ if after_key:+ composite["after"] = after_key++ return {+ "size": 0,+ "query": query,+ "aggs": {+ "files": {+ "composite": composite,+ "aggs": {+ "file_metadata": {+ "top_hits": {+ "size": 1,+ "_source": [+ "document_id",+ "filename",+ "mimetype",+ "file_size",+ "source_url",+ "owner",+ "owner_name",+ "owner_email",+ "connector_type",+ "embedding_model",+ "embedding_dimensions",+ "indexed_time",+ "allowed_users",+ "allowed_groups",+ "allowed_principal_labels",+ ],+ "sort": [{"indexed_time": {"order": "desc"}}],+ }+ },+ "chunk_count": {"value_count": {"field": "_id"}},+ },+ }+ },+ }++ def _build_terms_aggregation_for_chunk_count(+ self,+ query: dict[str, Any],+ page_size: int,+ offset: int,+ sort_order: str,+ ) -> dict[str, Any]:+ """+ Build a terms aggregation sorted by chunk_count sub-agg.++ Terms agg supports `order: { sub_agg: asc/desc }`. To implement+ offset-based pagination we request (offset + page_size) buckets and+ slice in Python — OpenSearch has no native offset for aggs.+ """+ return {+ "size": 0,+ "query": query,+ "aggs": {+ "files": {+ "terms": {+ "field": "filename",+ "size": offset + page_size,+ "order": {"chunk_count": sort_order},+ },+ "aggs": {+ "file_metadata": {+ "top_hits": {+ "size": 1,+ "_source": [+ "document_id",+ "filename",+ "mimetype",+ "file_size",+ "source_url",+ "owner",+ "owner_name",+ "owner_email",+ "connector_type",+ "embedding_model",+ "embedding_dimensions",+ "indexed_time",+ "allowed_users",+ "allowed_groups",+ "allowed_principal_labels",+ ],+ "sort": [{"indexed_time": {"order": "desc"}}],+ }+ },+ "chunk_count": {"value_count": {"field": "_id"}},+ },+ }+ },+ }++ def _parse_terms_buckets(self, result: dict[str, Any], offset: int = 0) -> list[dict[str, Any]]:+ """Parse terms agg buckets, applying offset slice. Returns files list."""+ buckets = result.get("aggregations", {}).get("files", {}).get("buckets", [])+ buckets = buckets[offset:]+ return self._buckets_to_files(buckets)++ async def _get_file_count(+ self, opensearch_client: Any, query: dict[str, Any]+ ) -> tuple[int, bool]:+ """Approximate unique-filename count via cardinality aggregation (O(1)).++ Returns (count, is_approximate). is_approximate is always True on+ success (cardinality agg is inherently approximate) and False when the+ aggregation fails and 0 is returned as a fallback.+ """+ body = {+ "size": 0,+ "query": query,+ "aggs": {+ "file_count": {+ "cardinality": {+ "field": "filename",+ "precision_threshold": 3000,+ }+ }+ },+ }+ try:+ result = await opensearch_client.search(index=get_index_name(), body=body)+ return result.get("aggregations", {}).get("file_count", {}).get("value", 0), True+ except Exception as e:+ logger.warning(+ "Failed to retrieve file count; pagination total will show 0", error=str(e)+ )+ return 0, True++ def _parse_composite_buckets(+ self, result: dict[str, Any]+ ) -> tuple[list[dict[str, Any]], dict | None]:+ """Parse composite agg buckets. Returns (files, next_after_key)."""+ agg = result.get("aggregations", {}).get("files", {})+ buckets = agg.get("buckets", [])+ next_after_key = agg.get("after_key")+ return self._buckets_to_files(buckets), next_after_key++ def _buckets_to_files(self, buckets: list[dict[str, Any]]) -> list[dict[str, Any]]:+ """Convert raw aggregation buckets (composite or terms) to file dicts."""+ files = []+ for bucket in buckets:+ hits = bucket.get("file_metadata", {}).get("hits", {}).get("hits", [])+ if not hits:+ continue+ source = hits[0].get("_source", {})+ files.append(+ {+ "filename": source.get("filename")+ or (+ bucket["key"].get("filename", "")+ if isinstance(bucket["key"], dict)+ else bucket.get("key", "")+ ),+ "document_id": source.get("document_id", ""),+ "mimetype": source.get("mimetype", ""),+ "file_size": source.get("file_size", 0),+ "source_url": source.get("source_url", ""),+ "owner": source.get("owner", ""),+ "owner_name": source.get("owner_name", ""),+ "owner_email": source.get("owner_email", ""),+ "connector_type": source.get("connector_type", ""),+ "embedding_model": source.get("embedding_model", ""),+ "embedding_dimensions": source.get("embedding_dimensions"),+ "indexed_time": source.get("indexed_time", ""),+ "chunk_count": bucket.get("chunk_count", {}).get("value", 0),+ "allowed_users": source.get("allowed_users", []),+ "allowed_groups": source.get("allowed_groups", []),+ "allowed_principal_labels": source.get("allowed_principal_labels", []),+ }+ )+ return files