Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: align source filter options with active table rows

Olfa Maslahacc4d1c2b89d
diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsx
index 9b88e855f..23b5596bd 100644
--- a/frontend/app/knowledge/page.tsx
+++ b/frontend/app/knowledge/page.tsx
@@ -29,6 +29,10 @@ import {
TooltipContent,
TooltipTrigger,
} from "@/components/ui/tooltip";
+import {
+ buildKnowledgeTableRows,
+ getKnowledgeFileIdentity,
+} from "@/lib/knowledge-table-state";
import { parseTimestampMs } from "@/lib/time-utils";
import {
DeleteConfirmationDialog,
@@ -208,21 +212,7 @@ function SearchPage() {
}, []);
const getFileIdentity = useCallback((file?: File) => {
- if (!file) {
- return "";
- }
-
- const normalizedFilename = file.filename?.trim();
- if (normalizedFilename) {
- return normalizedFilename;
- }
-
- const normalizedSourceUrl = file.source_url?.trim();
- if (normalizedSourceUrl) {
- return normalizedSourceUrl;
- }
-
- return "";
+ return getKnowledgeFileIdentity(file);
}, []);
const hasOpenragRefreshCueFromTasks = tasks.some((task) => {
@@ -262,68 +252,7 @@ function SearchPage() {
lastErrorRef.current = null;
}
}, [isError, error]);
- // Convert TaskFiles to File format and merge with backend results
- const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => {
- const normalizedFilename =
- taskFile.filename?.trim() ||
- taskFile.source_url?.trim() ||
- "Untitled source";
-
- return {
- filename: normalizedFilename,
- mimetype: taskFile.mimetype,
- source_url: taskFile.source_url || "",
- size: taskFile.size,
- connector_type: taskFile.connector_type,
- status: taskFile.status,
- error: taskFile.error,
- embedding_model: taskFile.embedding_model,
- embedding_dimensions: taskFile.embedding_dimensions,
- };
- });
- // Create a map of task files by filename for quick lookup
- const taskFileMap = new Map(
- taskFilesAsFiles.map((file) => [getFileIdentity(file), file]),
- );
- // Override backend files with task file status if they exist.
- // Keep openrag_docs rows sourced from indexed search results so
- // OpenRAG docs do not appear as pending in the table.
- const backendFiles = (searchData as File[]).map((file) => {
- if (file.connector_type === "openrag_docs") {
- return file;
- }
- const taskFile = taskFileMap.get(getFileIdentity(file));
- if (taskFile) {
- // Override backend file with task file data (includes status)
- return { ...file, ...taskFile };
- }
- return file;
- });
-
- const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => {
- // Ignore the synthetic refresh task row from docs URL ingestion.
- // The table should only show indexed docs, not orchestration task labels.
- if (
- taskFile.filename === "OpenRAG docs refresh" ||
- taskFile.source_url.includes("openr.ag")
- ) {
- return false;
- }
- // Do not render task-only openrag_docs placeholder rows in the table.
- // OpenRAG default docs should be represented only by indexed search results.
- if (taskFile.connector_type === "openrag_docs") {
- return false;
- }
- return (
- taskFile.status !== "active" &&
- !backendFiles.some(
- (backendFile) =>
- getFileIdentity(backendFile) === getFileIdentity(taskFile),
- )
- );
- });
- // Combine task files first, then backend files
- const fileResults = [...backendFiles, ...filteredTaskFiles];
+ const fileResults = buildKnowledgeTableRows(searchData as File[], taskFiles);
const gridRows = fileResults;
const gridRef = useRef<AgGridReact>(null);
diff --git a/frontend/components/knowledge-filter-panel.tsx b/frontend/components/knowledge-filter-panel.tsx
index b8f99f634..e003995c0 100644
--- a/frontend/components/knowledge-filter-panel.tsx
+++ b/frontend/components/knowledge-filter-panel.tsx
@@ -6,6 +6,10 @@ import { useCreateFilter } from "@/app/api/mutations/useCreateFilter";
import { useDeleteFilter } from "@/app/api/mutations/useDeleteFilter";
import { useUpdateFilter } from "@/app/api/mutations/useUpdateFilter";
import { useGetSearchAggregations } from "@/app/api/queries/useGetSearchAggregations";
+import {
+ type File as SearchFile,
+ useGetSearchQuery,
+} from "@/app/api/queries/useGetSearchQuery";
import {
type FilterColor,
FilterIconPopover,
@@ -25,6 +29,11 @@ import { MultiSelect } from "@/components/ui/multi-select";
import { Slider } from "@/components/ui/slider";
import { Textarea } from "@/components/ui/textarea";
import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context";
+import { useTask } from "@/contexts/task-context";
+import {
+ buildActiveSourceOptions,
+ buildKnowledgeTableRows,
+} from "@/lib/knowledge-table-state";
interface FacetBucket {
key: string;
@@ -59,6 +68,7 @@ export function KnowledgeFilterPanel() {
createMode,
endCreateMode,
} = useKnowledgeFilter();
+ const { files: taskFiles } = useTask();
const deleteFilterMutation = useDeleteFilter();
const updateFilterMutation = useUpdateFilter();
const createFilterMutation = useCreateFilter();
@@ -149,6 +159,10 @@ export function KnowledgeFilterPanel() {
gcTime: 5 * 60_000,
});
+ const { data: allSearchData = [] } = useGetSearchQuery("*", null, {
+ enabled: isPanelOpen,
+ });
+
useEffect(() => {
if (!aggregations) return;
const facets = {
@@ -160,6 +174,12 @@ export function KnowledgeFilterPanel() {
setAvailableFacets(facets);
}, [aggregations]);
+ const tableRows = buildKnowledgeTableRows(
+ allSearchData as SearchFile[],
+ taskFiles,
+ );
+ const sourceOptions = buildActiveSourceOptions(tableRows);
+
// Don't render if panel is closed or we don't have any data
if (!isPanelOpen || !parsedFilterData) return null;
@@ -337,11 +357,7 @@ export function KnowledgeFilterPanel() {
<div className="space-y-4">
<div className="space-y-2">
<MultiSelect
- options={(availableFacets.data_sources || []).map((bucket) => ({
- value: bucket.key,
- label: bucket.key,
- count: bucket.count,
- }))}
+ options={sourceOptions}
value={selectedFilters.data_sources}
onValueChange={(values) =>
handleFilterChange("data_sources", values)
diff --git a/frontend/lib/knowledge-table-state.ts b/frontend/lib/knowledge-table-state.ts
new file mode 100644
index 000000000..6eed1b2e0
--- /dev/null
+++ b/frontend/lib/knowledge-table-state.ts
@@ -0,0 +1,113 @@
+import type { File as SearchFile } from "@/app/api/queries/useGetSearchQuery";
+import type { TaskFile } from "@/contexts/task-context";
+
+export interface KnowledgeSourceOption {
+ value: string;
+ label: string;
+ count: number;
+}
+
+export function getKnowledgeFileIdentity(file?: {
+ filename?: string;
+ source_url?: string;
+}) {
+ if (!file) {
+ return "";
+ }
+
+ const normalizedFilename = file.filename?.trim();
+ if (normalizedFilename) {
+ return normalizedFilename;
+ }
+
+ const normalizedSourceUrl = file.source_url?.trim();
+ if (normalizedSourceUrl) {
+ return normalizedSourceUrl;
+ }
+
+ return "";
+}
+
+export function buildKnowledgeTableRows(
+ searchData: SearchFile[],
+ taskFiles: TaskFile[],
+): SearchFile[] {
+ const taskFilesAsFiles: SearchFile[] = taskFiles.map((taskFile) => {
+ const normalizedFilename =
+ taskFile.filename?.trim() ||
+ taskFile.source_url?.trim() ||
+ "Untitled source";
+
+ return {
+ filename: normalizedFilename,
+ mimetype: taskFile.mimetype,
+ source_url: taskFile.source_url || "",
+ size: taskFile.size,
+ connector_type: taskFile.connector_type,
+ status: taskFile.status,
+ error: taskFile.error,
+ embedding_model: taskFile.embedding_model,
+ embedding_dimensions: taskFile.embedding_dimensions,
+ };
+ });
+
+ const taskFileMap = new Map(
+ taskFilesAsFiles.map((file) => [getKnowledgeFileIdentity(file), file]),
+ );
+
+ const backendFiles = searchData.map((file) => {
+ if (file.connector_type === "openrag_docs") {
+ return file;
+ }
+ const taskFile = taskFileMap.get(getKnowledgeFileIdentity(file));
+ if (taskFile) {
+ return { ...file, ...taskFile };
+ }
+ return file;
+ });
+
+ const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => {
+ if (
+ taskFile.filename === "OpenRAG docs refresh" ||
+ taskFile.source_url.includes("openr.ag")
+ ) {
+ return false;
+ }
+ if (taskFile.connector_type === "openrag_docs") {
+ return false;
+ }
+ return (
+ taskFile.status !== "active" &&
+ !backendFiles.some(
+ (backendFile) =>
+ getKnowledgeFileIdentity(backendFile) ===
+ getKnowledgeFileIdentity(taskFile),
+ )
+ );
+ });
+
+ return [...backendFiles, ...filteredTaskFiles];
+}
+
+export function buildActiveSourceOptions(
+ rows: SearchFile[],
+): KnowledgeSourceOption[] {
+ const sourceCounts = rows
+ .filter((file) => (file.status || "active") === "active")
+ .reduce((acc, file) => {
+ const source = file.filename?.trim() || file.source_url?.trim();
+ if (!source) {
+ return acc;
+ }
+ acc.set(source, (acc.get(source) || 0) + 1);
+ return acc;
+ }, new Map<string, number>());
+
+ return Array.from(sourceCounts.entries())
+ .map(([source, count]) => ({
+ value: source,
+ label: source,
+ count,
+ }))
+ .sort((a, b) => a.label.localeCompare(b.label));
+}