Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: Knowledge table optimization (#2202) (#2203)

Jason Dongcb9244b54fe5
diff --git a/frontend/app/api/queries/useListFiles.ts b/frontend/app/api/queries/useListFiles.ts
index b5b06f385..7a28571c6 100644
--- a/frontend/app/api/queries/useListFiles.ts
+++ b/frontend/app/api/queries/useListFiles.ts
@@ -14,13 +14,16 @@ export interface ListFilesParams {
mimetype?: string;
owner?: string;
search?: string;
+ afterKey?: Record<string, unknown> | null; //composite pagination cursor, could be undefined in the beginning
}
export interface ListFilesResponse {
files: File[];
total: number;
+ is_approximate: boolean;
page: number;
page_size: number;
+ after_key: Record<string, unknown> | null;
}
export const useListFiles = (
@@ -41,8 +44,11 @@ export const useListFiles = (
if (params.mimetype) searchParams.set("mimetype", params.mimetype);
if (params.owner) searchParams.set("owner", params.owner);
if (params.search) searchParams.set("search", params.search);
+ if (params.afterKey)
+ searchParams.set("after_key", JSON.stringify(params.afterKey));
+
+ const url = `/api/v2/files?${searchParams.toString()}`; //v2 endpoint
- const url = `/api/files?${searchParams.toString()}`;
const response = await fetch(url);
if (!response.ok) {
@@ -56,7 +62,6 @@ export const useListFiles = (
const data = await response.json();
- // Map server response to File interface
const files: File[] = (data.files || []).map(
(f: Record<string, unknown>) => ({
filename: (f.filename as string) || "",
@@ -76,12 +81,16 @@ export const useListFiles = (
}),
);
- return {
+ const result: ListFilesResponse = {
files,
total: data.total || 0,
+ is_approximate: data.is_approximate ?? true,
page: data.page || 1,
page_size: data.page_size || 25,
+ after_key: data.after_key ?? null,
};
+
+ return result;
}
return useQuery(
diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsx
index a11408682..648f456cd 100644
--- a/frontend/app/knowledge/page.tsx
+++ b/frontend/app/knowledge/page.tsx
@@ -4,6 +4,7 @@ import { useQueryClient } from "@tanstack/react-query";
import {
type CheckboxSelectionCallbackParams,
type ColDef,
+ type ColumnState,
type GetRowIdParams,
themeQuartz,
type ValueFormatterParams,
@@ -32,6 +33,7 @@ import "@/components/AgGrid/agGridStyles.css";
import { toast } from "sonner";
import { KnowledgeActionsDropdown } from "@/components/knowledge-actions-dropdown";
import { KnowledgeBatchActionsBar } from "@/components/knowledge-batch-actions-bar";
+import { KnowledgePaginationFooter } from "@/components/knowledge-pagination-footer";
import { KnowledgeSearchBar } from "@/components/knowledge-search-bar";
import { KnowledgeSearchInput } from "@/components/knowledge-search-input";
import { RequirePermission } from "@/components/require-permission";
@@ -128,6 +130,17 @@ function getSourceIcon(connectorType?: string) {
}
}
+const AG_FIELD_TO_SORT_BY: Record<string, string> = {
+ filename: "filename",
+ size: "file_size",
+ mimetype: "mimetype",
+ owner: "owner",
+ chunkCount: "chunk_count",
+ embedding_model: "embedding_model",
+ embedding_dimensions: "embedding_dimensions",
+ status: "status",
+};
+
function SearchPage() {
const isCloudBrand = useIsCloudBrand();
const queryClient = useQueryClient();
@@ -148,8 +161,9 @@ function SearchPage() {
} = useKnowledgeFilter();
const [selectedRows, setSelectedRows] = useState<File[]>([]);
- // Keep the filter context aware of checked rows so "Create New Filter"
- // can pre-populate its sources from the current selection.
+ const [sortBy, setSortBy] = useState<string>("filename");
+ const [sortOrder, setSortOrder] = useState<"asc" | "desc">("asc");
+
useEffect(() => {
setSelectedSources(
selectedRows.flatMap((row) => (row.filename ? [row.filename] : [])),
@@ -170,6 +184,16 @@ function SearchPage() {
null,
);
+ const [currentPage, setCurrentPage] = useState(1);
+ const [currentPageSize, setCurrentPageSize] = useState(25);
+
+ const cursorCacheRef = useRef<Map<number, Record<string, unknown>>>(
+ null as any,
+ );
+ if (!cursorCacheRef.current) {
+ cursorCacheRef.current = new Map();
+ }
+
const handleOpenSyncDialog = useCallback(async () => {
setSyncPreview(null);
setSyncDialogOpen(true);
@@ -269,8 +293,6 @@ function SearchPage() {
[taskFiles, tasks],
);
- // Auto-open unified task panel only when a NEW task file transitions to failed
- // (skip initial failed files that already existed on page load).
useEffect(() => {
const failedFiles = taskFiles.filter((file) => file.status === "failed");
const seenKeys = seenFailedFileKeysRef.current;
@@ -311,8 +333,6 @@ function SearchPage() {
getFailedFileKey,
]);
- // Use server-side file listing for default/wildcard view; search otherwise.
- // Wildcard follows bar text or saved filter query (bar is cleared when a filter is picked).
const effectiveSearchText =
queryOverride.trim() || parsedFilterData?.query?.trim() || "";
const hasActiveFilters = parsedFilterData?.filters
@@ -325,12 +345,16 @@ function SearchPage() {
const {
data: listFilesData,
isLoading: isListFilesLoading,
+ isFetching: isListFilesFetching,
error: listFilesError,
isError: isListFilesError,
} = useListFiles(
{
- pageSize: 100,
- search: isWildcardQuery ? undefined : queryOverride,
+ page: currentPage,
+ pageSize: currentPageSize,
+ sortBy,
+ sortOrder,
+ afterKey: cursorCacheRef.current.get(currentPage) ?? null,
connectorType: listFilesFilterParam(
parsedFilterData?.filters?.connector_types,
),
@@ -355,14 +379,19 @@ function SearchPage() {
const { files: searchFiles, warnings: searchWarnings } =
searchData as SearchResult;
- // Merge data from whichever source is active
- const effectiveData: File[] = isWildcardQuery
- ? (listFilesData?.files ?? [])
- : searchFiles;
const isLoading = isWildcardQuery ? isListFilesLoading : isSearchLoading;
+
+ const isFetching = isWildcardQuery ? isListFilesFetching : isSearchLoading;
const error = isWildcardQuery ? listFilesError : searchError;
const isError = isWildcardQuery ? isListFilesError : isSearchError;
+ const effectiveData: File[] = isWildcardQuery
+ ? (listFilesData?.files ?? [])
+ : searchFiles.slice(
+ (currentPage - 1) * currentPageSize,
+ currentPage * currentPageSize,
+ );
+
const isOpenragDocsRow = useCallback((file?: File) => {
return (
file?.connector_type === "openrag_docs" ||
@@ -393,26 +422,6 @@ function SearchPage() {
return file?.owner_name?.trim() || file?.owner_email?.trim() || "—";
}, []);
- const normalizeSourceForSort = useCallback((value?: string): string => {
- const trimmed = (value || "").trim();
- if (!trimmed) {
- return "";
- }
-
- try {
- const parsed = new URL(trimmed);
- const hostname = parsed.hostname.toLowerCase();
- const pathname = parsed.pathname.replace(/\/+$/, "").toLowerCase();
- return `${hostname}${pathname}`;
- } catch {
- return trimmed
- .toLowerCase()
- .replace(/^https?:\/\//, "")
- .split(/[?#]/)[0]
- .replace(/\/+$/, "");
- }
- }, []);
-
const getStatusSortRank = useCallback((status?: File["status"]): number => {
switch (status) {
case "active":
@@ -469,14 +478,30 @@ function SearchPage() {
lastErrorRef.current = null;
}
}, [isError, error]);
- // Third arg: saved filter only — draft `parsedFilterData` (create mode) must still show task rows.
const fileResults = buildKnowledgeTableRows(
effectiveData,
taskFiles,
Boolean(selectedFilter),
);
- const gridRows = fileResults;
+ const serverTotal = isWildcardQuery
+ ? (listFilesData?.total ?? 0)
+ : searchFiles.length;
+ const gridRows: File[] = fileResults;
+ const totalPages = Math.max(1, Math.ceil(serverTotal / currentPageSize));
+
+ useEffect(() => {
+ cursorCacheRef.current = new Map();
+ setCurrentPage(1);
+ }, [effectiveSearchText]);
+
+ // when the server responds with an after_key for page N, cache it as the cursor for page N+1
+ useEffect(() => {
+ if (listFilesData?.after_key && listFilesData.page) {
+ const nextPage = listFilesData.page + 1;
+ cursorCacheRef.current.set(nextPage, listFilesData.after_key);
+ }
+ }, [listFilesData]);
const gridRef = useRef<AgGridReact>(null);
const gridReadyRef = useRef(false);
@@ -493,7 +518,27 @@ function SearchPage() {
return gridRef.current?.api ?? null;
}, []);
- // Re-run only when row identity/status changes, not on every list poll reference.
+ const onSortChanged = useCallback(() => {
+ const api = getGridApi();
+ if (!api) return;
+
+ const sortedCol: ColumnState | undefined = api
+ .getColumnState()
+ .find((col) => col.sort != null);
+
+ const newSortBy = sortedCol
+ ? (AG_FIELD_TO_SORT_BY[sortedCol.colId] ?? sortedCol.colId)
+ : "filename";
+ const newSortOrder: "asc" | "desc" =
+ sortedCol?.sort === "desc" ? "desc" : "asc";
+
+ // Changing sort invalidates all cursors; reset to page 1
+ cursorCacheRef.current = new Map();
+ setCurrentPage(1);
+ setSortBy(newSortBy);
+ setSortOrder(newSortOrder);
+ }, [getGridApi]);
+
const gridRowsSelectionKey = useMemo(
() =>
gridRows
@@ -521,19 +566,7 @@ function SearchPage() {
field: "filename",
headerName: "Source",
sortable: true,
- comparator: (valueA?: string, valueB?: string) => {
- const sourceA = normalizeSourceForSort(valueA);
- const sourceB = normalizeSourceForSort(valueB);
- if (sourceA === sourceB) {
- const fallbackA = (valueA || "").trim().toLowerCase();
- const fallbackB = (valueB || "").trim().toLowerCase();
- if (fallbackA === fallbackB) {
- return 0;
- }
- return fallbackA < fallbackB ? -1 : 1;
- }
- return sourceA < sourceB ? -1 : 1;
- },
+ comparator: () => 0,
checkboxSelection: (params: CheckboxSelectionCallbackParams<File>) =>
isDeletableKnowledgeRow(params?.data),
headerCheckboxSelection: true,
@@ -599,8 +632,7 @@ function SearchPage() {
headerName: "Size",
...(isCloudBrand ? { flex: 1, minWidth: 110 } : {}),
sortable: true,
- comparator: (valueA?: number, valueB?: number) =>
- (valueA || 0) - (valueB || 0),
+ comparator: () => 0,
valueFormatter: (params: ValueFormatterParams<File>) =>
params.value ? formatFileSize(params.value) : "-",
cellClass: isCloudBrand ? "text-muted-foreground" : undefined,
@@ -622,18 +654,14 @@ function SearchPage() {
sortable: true,
valueGetter: (params: ValueGetterParams<File>) =>
getOwnerLabel(params.data),
- comparator: (valueA?: string, valueB?: string) =>
- (valueA || "—").localeCompare(valueB || "—", undefined, {
- sensitivity: "base",
- }),
+ comparator: () => 0,
},
{
field: "chunkCount",
headerName: "Chunks",
...(isCloudBrand ? { flex: 0.9, minWidth: 95 } : {}),
sortable: true,
- comparator: (valueA?: number, valueB?: number) =>
- (valueA || 0) - (valueB || 0),
+ comparator: () => 0,
valueFormatter: (params: ValueFormatterParams<File>) =>
params.data?.chunkCount?.toString() || "-",
cellClass: isCloudBrand ? "text-muted-foreground" : undefined,
@@ -677,8 +705,7 @@ function SearchPage() {
headerName: "Dimensions",
...(isCloudBrand ? { flex: 0.9, minWidth: 110 } : { width: 110 }),
sortable: true,
- comparator: (valueA?: number, valueB?: number) =>
- (valueA || 0) - (valueB || 0),
+ comparator: () => 0,
cellRenderer: ({ data }: CustomCellRendererProps<File>) => (
<span className="text-xs text-muted-foreground">
{typeof data?.embedding_dimensions === "number"
@@ -879,15 +906,6 @@ function SearchPage() {
}
};
- // enables pagination in the grid
- const pagination = true;
-
- // sets 25 rows per page (default is 100)
- const paginationPageSize = 25;
-
- // allows the user to select the page size from a predefined list of page sizes
- const paginationPageSizeSelector = [10, 25, 50, 100];
-
return (
<>
<div className="flex flex-col h-full">
@@ -933,7 +951,7 @@ function SearchPage() {
</div>
) : (
/* Search Input Area */
- <div className="flex-1 flex items-center flex-shrink-0 flex-wrap-reverse gap-3 mb-6">
+ <div className="flex items-center flex-shrink-0 flex-wrap-reverse gap-3 mb-6">
<KnowledgeSearchInput />
<Button
@@ -1045,74 +1063,85 @@ function SearchPage() {
</div>
)}
{isCloudBrand ? (
- <AgGridReact
- className="w-full overflow-auto border"
- columnDefs={columnDefs as ColDef<File>[]}
- defaultColDef={defaultColDef}
- loading={isLoading || deleteDocumentMutation.isPending}
- ref={gridRef}
- theme={themeQuartz.withParams({ browserColorScheme: "inherit" })}
- rowData={gridRows}
- rowSelection="multiple"
- getRowId={(params: GetRowIdParams<File>) =>
- getFileIdentity(params.data)
- }
- isRowSelectable={(params) => isDeletableKnowledgeRow(params.data)}
- domLayout="normal"
- onGridReady={handleGridReady}
- onGridPreDestroyed={handleGridPreDestroyed}
- onSelectionChanged={onSelectionChanged}
- pagination={pagination}
- paginationPageSize={paginationPageSize}
- paginationPageSizeSelector={paginationPageSizeSelector}
- headerHeight={64}
- rowHeight={64}
- noRowsOverlayComponent={() => (
- <div className="text-center pb-[45px]">
- <div className="text-lg text-primary font-semibold">
- No knowledge
- </div>
- <div className="text-sm mt-1 text-muted-foreground">
- Add files from local or your preferred cloud.
+ <div className="flex-1 min-h-0 overflow-hidden">
+ <AgGridReact
+ className="w-full h-full border"
+ columnDefs={columnDefs as ColDef<File>[]}
+ defaultColDef={defaultColDef}
+ loading={isLoading || deleteDocumentMutation.isPending}
+ ref={gridRef}
+ theme={themeQuartz.withParams({ browserColorScheme: "inherit" })}
+ rowData={gridRows}
+ rowSelection="multiple"
+ getRowId={(params: GetRowIdParams<File>) =>
+ getFileIdentity(params.data)
+ }
+ isRowSelectable={(params) => isDeletableKnowledgeRow(params.data)}
+ domLayout="normal"
+ onGridReady={handleGridReady}
+ onGridPreDestroyed={handleGridPreDestroyed}
+ onSelectionChanged={onSelectionChanged}
+ onSortChanged={onSortChanged}
+ headerHeight={64}
+ rowHeight={64}
+ noRowsOverlayComponent={() => (
+ <div className="text-center pb-[45px]">
+ <div className="text-lg text-primary font-semibold">
+ No knowledge
+ </div>
+ <div className="text-sm mt-1 text-muted-foreground">
+ Add files from local or your preferred cloud.
+ </div>
</div>
- </div>
- )}
- />
+ )}
+ />
+ </div>
) : (
- <AgGridReact
- className="w-full overflow-auto"
- columnDefs={columnDefs as ColDef<File>[]}
- defaultColDef={defaultColDef}
- loading={isLoading || deleteDocumentMutation.isPending}
- ref={gridRef}
- theme={themeQuartz.withParams({ browserColorScheme: "inherit" })}
- rowData={gridRows}
- rowSelection="multiple"
- rowMultiSelectWithClick={false}
- suppressRowClickSelection={true}
- getRowId={(params: GetRowIdParams<File>) =>
- getFileIdentity(params.data)
- }
- isRowSelectable={(params) => isDeletableKnowledgeRow(params.data)}
- domLayout="normal"
- onGridReady={handleGridReady}
- onGridPreDestroyed={handleGridPreDestroyed}
- onSelectionChanged={onSelectionChanged}
- pagination={pagination}
- paginationPageSize={paginationPageSize}
- paginationPageSizeSelector={paginationPageSizeSelector}
- noRowsOverlayComponent={() => (
- <div className="text-center pb-[45px]">
- <div className="text-lg text-primary font-semibold">
- No knowledge
- </div>
- <div className="text-sm mt-1 text-muted-foreground">
- Add files from local or your preferred cloud.
+ <div className="flex-1 min-h-0 overflow-hidden">
+ <AgGridReact
+ className="w-full h-full"
+ columnDefs={columnDefs as ColDef<File>[]}
+ defaultColDef={defaultColDef}
+ loading={isLoading || deleteDocumentMutation.isPending}
+ ref={gridRef}
+ theme={themeQuartz.withParams({ browserColorScheme: "inherit" })}
+ rowData={gridRows}
+ rowSelection="multiple"
+ rowMultiSelectWithClick={false}
+ suppressRowClickSelection={true}
+ getRowId={(params: GetRowIdParams<File>) =>
+ getFileIdentity(params.data)
+ }
+ isRowSelectable={(params) => isDeletableKnowledgeRow(params.data)}
+ domLayout="normal"
+ onGridReady={handleGridReady}
+ onGridPreDestroyed={handleGridPreDestroyed}
+ onSelectionChanged={onSelectionChanged}
+ onSortChanged={onSortChanged}
+ noRowsOverlayComponent={() => (
+ <div className="text-center pb-[45px]">
+ <div className="text-lg text-primary font-semibold">
+ No knowledge
+ </div>
+ <div className="text-sm mt-1 text-muted-foreground">
+ Add files from local or your preferred cloud.
+ </div>
</div>
- </div>
- )}
- />
+ )}
+ />
+ </div>
)}
+
+ <KnowledgePaginationFooter
+ currentPage={currentPage}
+ currentPageSize={currentPageSize}
+ totalPages={totalPages}
+ serverTotal={serverTotal}
+ isLoading={isFetching}
+ cursorCacheRef={cursorCacheRef}
+ setCurrentPage={setCurrentPage}
+ setCurrentPageSize={setCurrentPageSize}
+ />
</div>
{/* Bulk Delete Confirmation Dialog */}
diff --git a/frontend/components/knowledge-pagination-footer.tsx b/frontend/components/knowledge-pagination-footer.tsx
new file mode 100644
index 000000000..5b1593682
--- /dev/null
+++ b/frontend/components/knowledge-pagination-footer.tsx
@@ -0,0 +1,109 @@
+import React from "react";
+
+export interface KnowledgePaginationFooterProps {
+ currentPage: number;
+ currentPageSize: number;
+ totalPages: number;
+ serverTotal: number;
+ isLoading?: boolean;
+ cursorCacheRef: React.RefObject<Map<number, Record<string, unknown>>>;
+ setCurrentPage: React.Dispatch<React.SetStateAction<number>>;
+ setCurrentPageSize: React.Dispatch<React.SetStateAction<number>>;
+}
+
+export function KnowledgePaginationFooter({
+ currentPage,
+ currentPageSize,
+ totalPages,
+ serverTotal,
+ isLoading = false,
+ cursorCacheRef,
+ setCurrentPage,
+ setCurrentPageSize,
+}: KnowledgePaginationFooterProps) {
+ const atFirst = currentPage <= 1;
+ const atLast = currentPage >= totalPages;
+
+ return (
+ <div
+ className="flex items-center justify-end gap-4 border-t border-border bg-background px-3 text-sm text-muted-foreground"
+ style={{ height: "var(--ag-pagination-panel-height, 48px)" }}
+ >
+ {/* page size */}
+ <label className="flex items-center gap-1.5 whitespace-nowrap">
+ Page Size:
+ <select
+ value={currentPageSize}
+ onChange={(e) => {
+ cursorCacheRef.current = new Map();
+ setCurrentPageSize(Number(e.target.value));
+ setCurrentPage(1);
+ }}
+ className="cursor-pointer rounded-none border border-border bg-background px-1 py-0.5 text-sm text-muted-foreground"
+ >
+ {[10, 25, 50, 100].map((n) => (
+ <option key={n} value={n}>
+ {n}
+ </option>
+ ))}
+ </select>
+ </label>
+
+ {/* row summary */}
+ <span className="whitespace-nowrap">
+ {serverTotal === 0
+ ? `0 of 0`
+ : `${(currentPage - 1) * currentPageSize + 1} to ${Math.min(
+ currentPage * currentPageSize,
+ serverTotal,
+ )} of ${serverTotal}`}
+ </span>
+
+ {/* nav buttons */}
+ <div className="flex items-center">
+ <button
+ type="button"
+ aria-label="Back to first page"
+ aria-disabled={atFirst || isLoading}
+ disabled={atFirst || isLoading}
+ onClick={() => {
+ if (!atFirst && !isLoading) {
+ cursorCacheRef.current = new Map();
+ setCurrentPage(1);
+ }
+ }}
+ className="-mr-2 flex h-10 w-10 items-center justify-center rounded text-lg leading-none disabled:cursor-default disabled:opacity-40"
+ >
+ <span className="pointer-events-none">«</span>
+ </button>
+ <button
+ type="button"
+ aria-label="Previous page"
+ aria-disabled={atFirst || isLoading}
+ disabled={atFirst || isLoading}
+ onClick={() => {
+ if (!atFirst && !isLoading) setCurrentPage((p) => p - 1);
+ }}
+ className="flex h-10 w-10 items-center justify-center rounded text-lg leading-none disabled:cursor-default disabled:opacity-40"
+ >
+ <span className="pointer-events-none">‹</span>
+ </button>
+ <span className="whitespace-nowrap px-2">
+ Page {currentPage} of {totalPages}
+ </span>
+ <button
+ type="button"
+ aria-label="Next page"
+ aria-disabled={atLast || isLoading}
+ disabled={atLast || isLoading}
+ onClick={() => {
+ if (!atLast && !isLoading) setCurrentPage((p) => p + 1);
+ }}
+ className="flex h-10 w-10 items-center justify-center rounded text-lg leading-none disabled:cursor-default disabled:opacity-40"
+ >
+ <span className="pointer-events-none">›</span>
+ </button>
+ </div>
+ </div>
+ );
+}
diff --git a/src/api/files.py b/src/api/files.py
index e3e0e9664..889d1e8de 100644
--- a/src/api/files.py
+++ b/src/api/files.py
@@ -1,8 +1,8 @@
"""
File listing and search API endpoints.
-Provides server-side file-level views over the chunk-based OpenSearch index,
-replacing the client-side chunk-to-file aggregation pattern.
+Provides server-side file-level views over the chunk-based OpenSearch index.
+Uses terms aggregation with in-memory sort/slice (v1).
"""
from fastapi import Depends, Query
diff --git a/src/api/v2/__init__.py b/src/api/v2/__init__.py
new file mode 100644
index 000000000..2e04b7cf1
--- /dev/null
+++ b/src/api/v2/__init__.py
@@ -0,0 +1,5 @@
+"""
+OpenRAG API v2
+
+Composite-aggregation endpoints — cursor-based pagination, O(page_size) per request.
+"""
diff --git a/src/api/v2/files.py b/src/api/v2/files.py
new file mode 100644
index 000000000..b6fc1b1f3
--- /dev/null
+++ b/src/api/v2/files.py
@@ -0,0 +1,119 @@
+"""
+File listing and search API endpoints — v2 (composite aggregation).
+
+Uses composite aggregation for true O(page_size) server-side pagination.
+Cursor-based: pass `after_key` (JSON-encoded) to advance pages.
+"""
+
+import json
+
+from fastapi import Depends, HTTPException, Query
+from fastapi.responses import JSONResponse
+
+from dependencies import get_current_user, get_file_service_v2
+from session_manager import User
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+def _parse_after_key(after_key: str | None) -> dict | None:
+ """Parse a JSON-encoded composite cursor.
+
+ Returns None when after_key is absent.
+ Raises HTTPException 400 when the value is present but is not valid JSON
+ or does not decode to a dict (e.g. a bare string or number is invalid).
+ """
+ if not after_key:
+ return None
+ try:
+ parsed = json.loads(after_key)
+ except (json.JSONDecodeError, ValueError) as err:
+ raise HTTPException(status_code=400, detail="after_key is not valid JSON") from err
+ if not isinstance(parsed, dict):
+ raise HTTPException(status_code=400, detail="after_key must be a JSON object")
+ return parsed
+
+
+async def list_files(
+ page: int = Query(
+ 1, ge=1, description="Page number (for display only; navigation uses after_key cursor)"
+ ),
+ page_size: int = Query(25, ge=1, le=500, description="Items per page"),
+ sort_by: str = Query("filename", description="Sort field"),
+ sort_order: str = Query("asc", pattern="^(asc|desc)$", description="Sort order"),
+ connector_type: str | None = Query(None, description="Filter by connector type"),
+ mimetype: str | None = Query(None, description="Filter by MIME type"),
+ owner: str | None = Query(None, description="Filter by owner"),
+ search: str | None = Query(None, description="Search filename"),
+ after_key: str | None = Query(None, description="Composite pagination cursor (JSON-encoded)"),
+ file_service=Depends(get_file_service_v2),
+ user: User = Depends(get_current_user),
+):
+ """List ingested files with composite-aggregation pagination, filtering, and sorting."""
+ parsed_after_key = _parse_after_key(after_key)
+
+ try:
+ result = await file_service.list_files(
+ user_id=user.user_id,
+ jwt_token=user.jwt_token,
+ page=page,
+ page_size=page_size,
+ sort_by=sort_by,
+ sort_order=sort_order,
+ connector_type=connector_type,
+ mimetype=mimetype,
+ owner=owner,
+ search=search,
+ after_key=parsed_after_key,
+ )
+ return JSONResponse(result)
+ except Exception as e:
+ logger.error("Failed to list files (v2)", error=str(e))
+ from utils.opensearch_utils import AUTH_ERROR_MESSAGE, is_opensearch_auth_error
+
+ if is_opensearch_auth_error(e):
+ return JSONResponse({"error": AUTH_ERROR_MESSAGE}, status_code=401)
+ return JSONResponse(
+ {"error": "Failed to list files"},
+ status_code=500,
+ )
+
+
+async def search_files(
+ q: str = Query(..., min_length=1, description="Search query"),
+ page: int = Query(1, ge=1, description="Page number"),
+ page_size: int = Query(25, ge=1, le=500, description="Items per page"),
+ connector_type: str | None = Query(None, description="Filter by connector type"),
+ mimetype: str | None = Query(None, description="Filter by MIME type"),
+ owner: str | None = Query(None, description="Filter by owner"),
+ after_key: str | None = Query(None, description="Composite pagination cursor (JSON-encoded)"),
+ file_service=Depends(get_file_service_v2),
+ user: User = Depends(get_current_user),
+):
+ """Search files by name with fuzzy/partial matching (v2 — composite pagination)."""
+ parsed_after_key = _parse_after_key(after_key)
+
+ try:
+ result = await file_service.search_files(
+ user_id=user.user_id,
+ jwt_token=user.jwt_token,
+ query=q,
+ page=page,
+ page_size=page_size,
+ connector_type=connector_type,
+ mimetype=mimetype,
+ owner=owner,
+ after_key=parsed_after_key,
+ )
+ return JSONResponse(result)
+ except Exception as e:
+ logger.error("Failed to search files (v2)", error=str(e))
+ from utils.opensearch_utils import AUTH_ERROR_MESSAGE, is_opensearch_auth_error
+
+ if is_opensearch_auth_error(e):
+ return JSONResponse({"error": AUTH_ERROR_MESSAGE}, status_code=401)
+ return JSONResponse(
+ {"error": "Failed to search files"},
+ status_code=500,
+ )
diff --git a/src/app/container.py b/src/app/container.py
index a7a7918c5..27d538488 100644
--- a/src/app/container.py
+++ b/src/app/container.py
@@ -23,6 +23,7 @@ from services.dls_principal_service import DLSPrincipalService
from services.docling_polling_service import DoclingPollingService
from services.document_index_writer import DocumentIndexWriter
from services.document_service import DocumentService
+from services.file_service_v2 import FileServiceV2
from services.flows_service import FlowsService
from services.group_acl_service import GroupACLService
from services.ingest_preview_service import IngestPreviewService
@@ -213,6 +214,8 @@ async def initialize_services():
session_ownership_service._session_factory = _lazy_session_factory
conversation_persistence._session_factory = _lazy_session_factory
+ file_service_v2 = FileServiceV2(session_manager=session_manager)
+
return {
"document_service": document_service,
"search_service": search_service,
@@ -237,4 +240,5 @@ async def initialize_services():
"ingest_preview_service": ingest_preview_service,
"rbac_service": rbac_service,
"workspace_config_service": workspace_config_service,
+ "file_service_v2": file_service_v2,
}
diff --git a/src/app/routes/internal.py b/src/app/routes/internal.py
index 25e2492b1..c9703e833 100644
--- a/src/app/routes/internal.py
+++ b/src/app/routes/internal.py
@@ -33,6 +33,7 @@ from api import (
from api import keys as api_keys
from api.health import health_check, opensearch_health_ready
from api.schemas.tasks import ErrorResponse, TaskRetryResponse
+from api.v2 import files as files_v2
from connectors.registry import get_connector_classes
@@ -121,10 +122,14 @@ def register_internal_routes(app: FastAPI):
# Search endpoint
app.add_api_route("/search", search.search, methods=["POST"], tags=["internal"])
- # File listing/search endpoints
+ # File listing/search endpoints (v1 — terms-agg, in-memory sort)
app.add_api_route("/files", files.list_files, methods=["GET"], tags=["internal"])
app.add_api_route("/files/search", files.search_files, methods=["GET"], tags=["internal"])
+ # File listing/search endpoints (v2 — composite-agg, cursor pagination)
+ app.add_api_route("/v2/files/search", files_v2.search_files, methods=["GET"], tags=["internal"])
+ app.add_api_route("/v2/files", files_v2.list_files, methods=["GET"], tags=["internal"])
+
# Knowledge Filter endpoints
app.add_api_route(
"/knowledge-filter",
diff --git a/src/dependencies.py b/src/dependencies.py
index 8c4334025..b9fb51654 100644
--- a/src/dependencies.py
+++ b/src/dependencies.py
@@ -66,6 +66,7 @@ __all__ = [
"get_api_key_service",
"get_api_key_user_async",
"get_auth_service",
+ "get_file_service_v2",
"get_chat_service",
"get_connector_service",
"get_current_user",
@@ -176,6 +177,10 @@ def get_api_key_service(services: dict = Depends(get_services)):
return services["api_key_service"]
+def get_file_service_v2(services: dict = Depends(get_services)):
+ return services["file_service_v2"]
+
+
def get_flows_service(services: dict = Depends(get_services)):
return services["flows_service"]
diff --git a/src/services/file_service.py b/src/services/file_service.py
index ce3f86786..dc5f620d7 100644
--- a/src/services/file_service.py
+++ b/src/services/file_service.py
@@ -238,8 +238,9 @@ class FileService:
reverse = sort_order.lower() == "desc"
- return sorted(
- files,
- key=lambda f: f.get(sort_by) or "",
- reverse=reverse,
- )
+ _numeric_sort_fields = {"file_size", "chunk_count"}
+
+ def _sort_key(f):
+ return f.get(sort_by) or (0 if sort_by in _numeric_sort_fields else "")
+
+ return sorted(files, key=_sort_key, reverse=reverse)
diff --git a/src/services/file_service_v2.py b/src/services/file_service_v2.py
new file mode 100644
index 000000000..6a4607dd0
--- /dev/null
+++ b/src/services/file_service_v2.py
@@ -0,0 +1,467 @@
+"""
+File service v2 — composite aggregation pagination.
+
+Uses OpenSearch composite aggregation for O(page_size) server-side
+pagination. Only page_size buckets are processed per request regardless of
+total file count.
+
+Sort-field notes
+----------------
+* Fields in _COMPOSITE_SORT_FIELDS are handled entirely by the composite agg —
+ ordering is globally correct and cursor-paginated.
+* "owner" is a real keyword field so it lives in _COMPOSITE_SORT_FIELDS.
+* "chunk_count" is a derived metric (value_count sub-agg). Composite aggs
+ cannot order by sub-aggregation metrics, so sorting by chunk_count falls back
+ to a classic terms aggregation sorted by the sub-agg, with offset-based
+ pagination (_build_terms_aggregation_for_chunk_count). This is O(from+size)
+ at the shard level but gives a globally correct sort.
+"""
+
+from typing import Any
+
+from config.settings import get_index_name
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+_COMPOSITE_SORT_FIELDS: dict[str, str] = {
+ "filename": "filename",
+ "file_size": "file_size",
+ "mimetype": "mimetype",
+ "indexed_time": "indexed_time",
+ "connector_type": "connector_type",
+ "embedding_model": "embedding_model",
+ "owner": "owner",
+}
+
+
+class FileServiceV2:
+ """File-level views via composite aggregation (v2 — cursor pagination)."""
+
+ def __init__(self, session_manager=None):
+ self.session_manager = session_manager
+
+ async def list_files(
+ self,
+ user_id: str,
+ jwt_token: str = None,
+ page: int = 1,
+ page_size: int = 25,
+ sort_by: str = "filename",
+ sort_order: str = "asc",
+ connector_type: str | None = None,
+ mimetype: str | None = None,
+ owner: str | None = None,
+ search: str | None = None,
+ after_key: dict | None = None,
+ ) -> dict[str, Any]:
+ """
+ List files with server-side pagination via composite aggregation.
+
+ Cost is O(page_size) per request (as opposed to returning all unique
+ file sizes from OpenSearch). Returns after_key for the next page
+ (None when on the last page), plus an approximate total from a
+ cardinality aggregation.
+
+ Exception: when sort_by="chunk_count" a terms aggregation is used
+ instead (composite aggs cannot order by sub-agg metrics). Pagination
+ is offset-based in that path; after_key is always None.
+ """
+ opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)
+
+ query = self._build_filter_query(user_id, connector_type, mimetype, owner, search)
+ total, is_approximate = await self._get_file_count(opensearch_client, query)
+
+ if sort_by == "chunk_count":
+ return await self._list_files_by_chunk_count(
+ opensearch_client=opensearch_client,
+ query=query,
+ page=page,
+ page_size=page_size,
+ sort_order=sort_order,
+ total=total,
+ is_approximate=is_approximate,
+ )
+
+ if page > 1 and after_key is None:
+ return {
+ "files": [],
+ "total": total,
+ "is_approximate": is_approximate,
+ "page": page,
+ "page_size": page_size,
+ "after_key": None,
+ }
+
+ composite_sort_field = _COMPOSITE_SORT_FIELDS.get(sort_by, "filename")
+
+ agg_body = self._build_composite_aggregation(
+ query=query,
+ page_size=page_size,
+ sort_field=composite_sort_field,
+ sort_order=sort_order,
+ after_key=after_key,
+ )
+
+ try:
+ result = await opensearch_client.search(
+ index=get_index_name(),
+ body=agg_body,
+ )
+ except Exception as e:
+ logger.error("Failed to list files (v2)", error=str(e))
+ from utils.opensearch_utils import is_opensearch_auth_error
+
+ if is_opensearch_auth_error(e):
+ raise
+ return {
+ "files": [],
+ "total": 0,
+ "is_approximate": False,
+ "page": page,
+ "page_size": page_size,
+ "after_key": None,
+ }
+
+ raw_bucket_count = len(result.get("aggregations", {}).get("files", {}).get("buckets", []))
+ files, next_after_key = self._parse_composite_buckets(result)
+
+ if raw_bucket_count < page_size: # final page, no after_key
+ next_after_key = None
+
+ return {
+ "files": files,
+ "total": total,
+ "is_approximate": is_approximate,
+ "page": page,
+ "page_size": page_size,
+ "after_key": next_after_key,
+ }
+
+ async def _list_files_by_chunk_count(
+ self,
+ opensearch_client: Any,
+ query: dict[str, Any],
+ page: int,
+ page_size: int,
+ sort_order: str,
+ total: int,
+ is_approximate: bool,
+ ) -> dict[str, Any]:
+ """
+ List files sorted globally by chunk_count using a terms aggregation.
+
+ Terms aggs support ordering by sub-aggregation metrics, which composite
+ aggs do not. Pagination is offset-based (from = (page-1)*page_size).
+ """
+ offset = (page - 1) * page_size
+ agg_body = self._build_terms_aggregation_for_chunk_count(
+ query=query,
+ page_size=page_size,
+ offset=offset,
+ sort_order=sort_order,
+ )
+
+ try:
+ result = await opensearch_client.search(
+ index=get_index_name(),
+ body=agg_body,
+ )
+ except Exception as e:
+ logger.error("Failed to list files by chunk_count (v2)", error=str(e))
+ from utils.opensearch_utils import is_opensearch_auth_error
+
+ if is_opensearch_auth_error(e):
+ raise
+ return {
+ "files": [],
+ "total": 0,
+ "page": page,
+ "page_size": page_size,
+ "after_key": None,
+ }
+
+ files = self._parse_terms_buckets(result, offset=offset)
+
+ return {
+ "files": files,
+ "total": total,
+ "is_approximate": is_approximate,
+ "page": page,
+ "page_size": page_size,
+ "after_key": None, # offset pagination; no cursor
+ }
+
+ async def search_files(
+ self,
+ user_id: str,
+ jwt_token: str = None,
+ query: str = "",
+ page: int = 1,
+ page_size: int = 25,
+ connector_type: str | None = None,
+ mimetype: str | None = None,
+ owner: str | None = None,
+ after_key: dict | None = None,
+ ) -> dict[str, Any]:
+ """Search files by name with fuzzy/prefix matching."""
+ return await self.list_files(
+ user_id=user_id,
+ jwt_token=jwt_token,
+ page=page,
+ page_size=page_size,
+ sort_by="filename",
+ sort_order="asc",
+ connector_type=connector_type,
+ mimetype=mimetype,
+ owner=owner,
+ search=query,
+ after_key=after_key,
+ )
+
+ def _build_filter_query(
+ self,
+ user_id: str,
+ connector_type: str | None = None,
+ mimetype: str | None = None,
+ owner: str | None = None,
+ search: str | None = None,
+ ) -> dict[str, Any]:
+ must = []
+ filter_clauses = []
+
+ if connector_type:
+ filter_clauses.append({"term": {"connector_type": connector_type}})
+ if mimetype:
+ filter_clauses.append({"term": {"mimetype": mimetype}})
+ if owner:
+ filter_clauses.append({"term": {"owner": owner}})
+
+ if search:
+ must.append(
+ {
+ "bool": {
+ "should": [
+ {"wildcard": {"filename": {"value": f"*{search.lower()}*"}}},
+ {"prefix": {"filename": search.lower()}},
+ ],
+ "minimum_should_match": 1,
+ }
+ }
+ )
+
+ query: dict[str, Any] = {"bool": {"filter": filter_clauses}}
+ if must:
+ query["bool"]["must"] = must
+ return query
+
+ def _build_composite_aggregation(
+ self,
+ query: dict[str, Any],
+ page_size: int,
+ sort_field: str,
+ sort_order: str,
+ after_key: dict | None,
+ ) -> dict[str, Any]:
+ composite: dict[str, Any] = {
+ "size": page_size,
+ "sources": [
+ {
+ sort_field: {
+ "terms": {
+ "field": sort_field,
+ "order": sort_order,
+ "missing_bucket": True,
+ }
+ }
+ },
+ *(
+ [
+ {
+ "filename_tiebreak": {
+ "terms": {
+ "field": "filename",
+ "order": sort_order,
+ "missing_bucket": True,
+ }
+ }
+ }
+ ]
+ if sort_field != "filename"
+ else []
+ ),
+ ],
+ }
+
+ if after_key:
+ composite["after"] = after_key
+
+ return {
+ "size": 0,
+ "query": query,
+ "aggs": {
+ "files": {
+ "composite": composite,
+ "aggs": {
+ "file_metadata": {
+ "top_hits": {
+ "size": 1,
+ "_source": [
+ "document_id",
+ "filename",
+ "mimetype",
+ "file_size",
+ "source_url",
+ "owner",
+ "owner_name",
+ "owner_email",
+ "connector_type",
+ "embedding_model",
+ "embedding_dimensions",
+ "indexed_time",
+ "allowed_users",
+ "allowed_groups",
+ "allowed_principal_labels",
+ ],
+ "sort": [{"indexed_time": {"order": "desc"}}],
+ }
+ },
+ "chunk_count": {"value_count": {"field": "_id"}},
+ },
+ }
+ },
+ }
+
+ def _build_terms_aggregation_for_chunk_count(
+ self,
+ query: dict[str, Any],
+ page_size: int,
+ offset: int,
+ sort_order: str,
+ ) -> dict[str, Any]:
+ """
+ Build a terms aggregation sorted by chunk_count sub-agg.
+
+ Terms agg supports `order: { sub_agg: asc/desc }`. To implement
+ offset-based pagination we request (offset + page_size) buckets and
+ slice in Python — OpenSearch has no native offset for aggs.
+ """
+ return {
+ "size": 0,
+ "query": query,
+ "aggs": {
+ "files": {
+ "terms": {
+ "field": "filename",
+ "size": offset + page_size,
+ "order": {"chunk_count": sort_order},
+ },
+ "aggs": {
+ "file_metadata": {
+ "top_hits": {
+ "size": 1,
+ "_source": [
+ "document_id",
+ "filename",
+ "mimetype",
+ "file_size",
+ "source_url",
+ "owner",
+ "owner_name",
+ "owner_email",
+ "connector_type",
+ "embedding_model",
+ "embedding_dimensions",
+ "indexed_time",
+ "allowed_users",
+ "allowed_groups",
+ "allowed_principal_labels",
+ ],
+ "sort": [{"indexed_time": {"order": "desc"}}],
+ }
+ },
+ "chunk_count": {"value_count": {"field": "_id"}},
+ },
+ }
+ },
+ }
+
+ def _parse_terms_buckets(self, result: dict[str, Any], offset: int = 0) -> list[dict[str, Any]]:
+ """Parse terms agg buckets, applying offset slice. Returns files list."""
+ buckets = result.get("aggregations", {}).get("files", {}).get("buckets", [])
+ buckets = buckets[offset:]
+ return self._buckets_to_files(buckets)
+
+ async def _get_file_count(
+ self, opensearch_client: Any, query: dict[str, Any]
+ ) -> tuple[int, bool]:
+ """Approximate unique-filename count via cardinality aggregation (O(1)).
+
+ Returns (count, is_approximate). is_approximate is always True on
+ success (cardinality agg is inherently approximate) and False when the
+ aggregation fails and 0 is returned as a fallback.
+ """
+ body = {
+ "size": 0,
+ "query": query,
+ "aggs": {
+ "file_count": {
+ "cardinality": {
+ "field": "filename",
+ "precision_threshold": 3000,
+ }
+ }
+ },
+ }
+ try:
+ result = await opensearch_client.search(index=get_index_name(), body=body)
+ return result.get("aggregations", {}).get("file_count", {}).get("value", 0), True
+ except Exception as e:
+ logger.warning(
+ "Failed to retrieve file count; pagination total will show 0", error=str(e)
+ )
+ return 0, True
+
+ def _parse_composite_buckets(
+ self, result: dict[str, Any]
+ ) -> tuple[list[dict[str, Any]], dict | None]:
+ """Parse composite agg buckets. Returns (files, next_after_key)."""
+ agg = result.get("aggregations", {}).get("files", {})
+ buckets = agg.get("buckets", [])
+ next_after_key = agg.get("after_key")
+ return self._buckets_to_files(buckets), next_after_key
+
+ def _buckets_to_files(self, buckets: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ """Convert raw aggregation buckets (composite or terms) to file dicts."""
+ files = []
+ for bucket in buckets:
+ hits = bucket.get("file_metadata", {}).get("hits", {}).get("hits", [])
+ if not hits:
+ continue
+ source = hits[0].get("_source", {})
+ files.append(
+ {
+ "filename": source.get("filename")
+ or (
+ bucket["key"].get("filename", "")
+ if isinstance(bucket["key"], dict)
+ else bucket.get("key", "")
+ ),
+ "document_id": source.get("document_id", ""),
+ "mimetype": source.get("mimetype", ""),
+ "file_size": source.get("file_size", 0),
+ "source_url": source.get("source_url", ""),
+ "owner": source.get("owner", ""),
+ "owner_name": source.get("owner_name", ""),
+ "owner_email": source.get("owner_email", ""),
+ "connector_type": source.get("connector_type", ""),
+ "embedding_model": source.get("embedding_model", ""),
+ "embedding_dimensions": source.get("embedding_dimensions"),
+ "indexed_time": source.get("indexed_time", ""),
+ "chunk_count": bucket.get("chunk_count", {}).get("value", 0),
+ "allowed_users": source.get("allowed_users", []),
+ "allowed_groups": source.get("allowed_groups", []),
+ "allowed_principal_labels": source.get("allowed_principal_labels", []),
+ }
+ )
+ return files