OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: align source filter options with active table rows
Olfa Maslahacc4d1c2b89d
diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsxindex 9b88e855f..23b5596bd 100644--- a/frontend/app/knowledge/page.tsx+++ b/frontend/app/knowledge/page.tsx@@ -29,6 +29,10 @@ import {TooltipContent,TooltipTrigger,} from "@/components/ui/tooltip";+import {+ buildKnowledgeTableRows,+ getKnowledgeFileIdentity,+} from "@/lib/knowledge-table-state";import { parseTimestampMs } from "@/lib/time-utils";import {DeleteConfirmationDialog,@@ -208,21 +212,7 @@ function SearchPage() {}, []);const getFileIdentity = useCallback((file?: File) => {- if (!file) {- return "";- }-- const normalizedFilename = file.filename?.trim();- if (normalizedFilename) {- return normalizedFilename;- }-- const normalizedSourceUrl = file.source_url?.trim();- if (normalizedSourceUrl) {- return normalizedSourceUrl;- }-- return "";+ return getKnowledgeFileIdentity(file);}, []);const hasOpenragRefreshCueFromTasks = tasks.some((task) => {@@ -262,68 +252,7 @@ function SearchPage() {lastErrorRef.current = null;}}, [isError, error]);- // Convert TaskFiles to File format and merge with backend results- const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => {- const normalizedFilename =- taskFile.filename?.trim() ||- taskFile.source_url?.trim() ||- "Untitled source";-- return {- filename: normalizedFilename,- mimetype: taskFile.mimetype,- source_url: taskFile.source_url || "",- size: taskFile.size,- connector_type: taskFile.connector_type,- status: taskFile.status,- error: taskFile.error,- embedding_model: taskFile.embedding_model,- embedding_dimensions: taskFile.embedding_dimensions,- };- });- // Create a map of task files by filename for quick lookup- const taskFileMap = new Map(- taskFilesAsFiles.map((file) => [getFileIdentity(file), file]),- );- // Override backend files with task file status if they exist.- // Keep openrag_docs rows sourced from indexed search results so- // OpenRAG docs do not appear as pending in the table.- const backendFiles = (searchData as File[]).map((file) => {- if (file.connector_type === "openrag_docs") {- return file;- }- const taskFile = taskFileMap.get(getFileIdentity(file));- if (taskFile) {- // Override backend file with task file data (includes status)- return { ...file, ...taskFile };- }- return file;- });-- const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => {- // Ignore the synthetic refresh task row from docs URL ingestion.- // The table should only show indexed docs, not orchestration task labels.- if (- taskFile.filename === "OpenRAG docs refresh" ||- taskFile.source_url.includes("openr.ag")- ) {- return false;- }- // Do not render task-only openrag_docs placeholder rows in the table.- // OpenRAG default docs should be represented only by indexed search results.- if (taskFile.connector_type === "openrag_docs") {- return false;- }- return (- taskFile.status !== "active" &&- !backendFiles.some(- (backendFile) =>- getFileIdentity(backendFile) === getFileIdentity(taskFile),- )- );- });- // Combine task files first, then backend files- const fileResults = [...backendFiles, ...filteredTaskFiles];+ const fileResults = buildKnowledgeTableRows(searchData as File[], taskFiles);const gridRows = fileResults;const gridRef = useRef<AgGridReact>(null);diff --git a/frontend/components/knowledge-filter-panel.tsx b/frontend/components/knowledge-filter-panel.tsxindex b8f99f634..e003995c0 100644--- a/frontend/components/knowledge-filter-panel.tsx+++ b/frontend/components/knowledge-filter-panel.tsx@@ -6,6 +6,10 @@ import { useCreateFilter } from "@/app/api/mutations/useCreateFilter";import { useDeleteFilter } from "@/app/api/mutations/useDeleteFilter";import { useUpdateFilter } from "@/app/api/mutations/useUpdateFilter";import { useGetSearchAggregations } from "@/app/api/queries/useGetSearchAggregations";+import {+ type File as SearchFile,+ useGetSearchQuery,+} from "@/app/api/queries/useGetSearchQuery";import {type FilterColor,FilterIconPopover,@@ -25,6 +29,11 @@ import { MultiSelect } from "@/components/ui/multi-select";import { Slider } from "@/components/ui/slider";import { Textarea } from "@/components/ui/textarea";import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context";+import { useTask } from "@/contexts/task-context";+import {+ buildActiveSourceOptions,+ buildKnowledgeTableRows,+} from "@/lib/knowledge-table-state";interface FacetBucket {key: string;@@ -59,6 +68,7 @@ export function KnowledgeFilterPanel() {createMode,endCreateMode,} = useKnowledgeFilter();+ const { files: taskFiles } = useTask();const deleteFilterMutation = useDeleteFilter();const updateFilterMutation = useUpdateFilter();const createFilterMutation = useCreateFilter();@@ -149,6 +159,10 @@ export function KnowledgeFilterPanel() {gcTime: 5 * 60_000,});+ const { data: allSearchData = [] } = useGetSearchQuery("*", null, {+ enabled: isPanelOpen,+ });+useEffect(() => {if (!aggregations) return;const facets = {@@ -160,6 +174,12 @@ export function KnowledgeFilterPanel() {setAvailableFacets(facets);}, [aggregations]);+ const tableRows = buildKnowledgeTableRows(+ allSearchData as SearchFile[],+ taskFiles,+ );+ const sourceOptions = buildActiveSourceOptions(tableRows);+// Don't render if panel is closed or we don't have any dataif (!isPanelOpen || !parsedFilterData) return null;@@ -337,11 +357,7 @@ export function KnowledgeFilterPanel() {<div className="space-y-4"><div className="space-y-2"><MultiSelect- options={(availableFacets.data_sources || []).map((bucket) => ({- value: bucket.key,- label: bucket.key,- count: bucket.count,- }))}+ options={sourceOptions}value={selectedFilters.data_sources}onValueChange={(values) =>handleFilterChange("data_sources", values)diff --git a/frontend/lib/knowledge-table-state.ts b/frontend/lib/knowledge-table-state.tsnew file mode 100644index 000000000..6eed1b2e0--- /dev/null+++ b/frontend/lib/knowledge-table-state.ts@@ -0,0 +1,113 @@+import type { File as SearchFile } from "@/app/api/queries/useGetSearchQuery";+import type { TaskFile } from "@/contexts/task-context";++export interface KnowledgeSourceOption {+ value: string;+ label: string;+ count: number;+}++export function getKnowledgeFileIdentity(file?: {+ filename?: string;+ source_url?: string;+}) {+ if (!file) {+ return "";+ }++ const normalizedFilename = file.filename?.trim();+ if (normalizedFilename) {+ return normalizedFilename;+ }++ const normalizedSourceUrl = file.source_url?.trim();+ if (normalizedSourceUrl) {+ return normalizedSourceUrl;+ }++ return "";+}++export function buildKnowledgeTableRows(+ searchData: SearchFile[],+ taskFiles: TaskFile[],+): SearchFile[] {+ const taskFilesAsFiles: SearchFile[] = taskFiles.map((taskFile) => {+ const normalizedFilename =+ taskFile.filename?.trim() ||+ taskFile.source_url?.trim() ||+ "Untitled source";++ return {+ filename: normalizedFilename,+ mimetype: taskFile.mimetype,+ source_url: taskFile.source_url || "",+ size: taskFile.size,+ connector_type: taskFile.connector_type,+ status: taskFile.status,+ error: taskFile.error,+ embedding_model: taskFile.embedding_model,+ embedding_dimensions: taskFile.embedding_dimensions,+ };+ });++ const taskFileMap = new Map(+ taskFilesAsFiles.map((file) => [getKnowledgeFileIdentity(file), file]),+ );++ const backendFiles = searchData.map((file) => {+ if (file.connector_type === "openrag_docs") {+ return file;+ }+ const taskFile = taskFileMap.get(getKnowledgeFileIdentity(file));+ if (taskFile) {+ return { ...file, ...taskFile };+ }+ return file;+ });++ const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => {+ if (+ taskFile.filename === "OpenRAG docs refresh" ||+ taskFile.source_url.includes("openr.ag")+ ) {+ return false;+ }+ if (taskFile.connector_type === "openrag_docs") {+ return false;+ }+ return (+ taskFile.status !== "active" &&+ !backendFiles.some(+ (backendFile) =>+ getKnowledgeFileIdentity(backendFile) ===+ getKnowledgeFileIdentity(taskFile),+ )+ );+ });++ return [...backendFiles, ...filteredTaskFiles];+}++export function buildActiveSourceOptions(+ rows: SearchFile[],+): KnowledgeSourceOption[] {+ const sourceCounts = rows+ .filter((file) => (file.status || "active") === "active")+ .reduce((acc, file) => {+ const source = file.filename?.trim() || file.source_url?.trim();+ if (!source) {+ return acc;+ }+ acc.set(source, (acc.get(source) || 0) + 1);+ return acc;+ }, new Map<string, number>());++ return Array.from(sourceCounts.entries())+ .map(([source, count]) => ({+ value: source,+ label: source,+ count,+ }))+ .sort((a, b) => a.label.localeCompare(b.label));+}