OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
Normalize file identity and OpenRAG/Langflow changes
Edwin Jose33f9feb45f15
diff --git a/frontend/app/api/queries/useGetSearchQuery.ts b/frontend/app/api/queries/useGetSearchQuery.tsindex e0b96f719..88bb9d45e 100644--- a/frontend/app/api/queries/useGetSearchQuery.ts+++ b/frontend/app/api/queries/useGetSearchQuery.ts@@ -71,6 +71,20 @@ export const useGetSearchQuery = () => {const queryClient = useQueryClient();+ const getFileIdentity = (chunk: ChunkResult): string => {+ const normalizedFilename = chunk.filename?.trim();+ if (normalizedFilename) {+ return normalizedFilename;+ }++ const normalizedSourceUrl = chunk.source_url?.trim();+ if (normalizedSourceUrl) {+ return normalizedSourceUrl;+ }++ return "Untitled source";+ };+// Normalize the query to match what will actually be searchedconst effectiveQuery = query || queryData?.query || "*";@@ -134,7 +148,8 @@ export const useGetSearchQuery = (>();(data.results || []).forEach((chunk: ChunkResult) => {- const existing = fileMap.get(chunk.filename);+ const fileIdentity = getFileIdentity(chunk);+ const existing = fileMap.get(fileIdentity);if (existing) {existing.chunks.push(chunk);existing.totalScore += chunk.score;@@ -148,8 +163,8 @@ export const useGetSearchQuery = (existing.embedding_dimensions = chunk.embedding_dimensions;}} else {- fileMap.set(chunk.filename, {- filename: chunk.filename,+ fileMap.set(fileIdentity, {+ filename: fileIdentity,mimetype: chunk.mimetype,chunks: [chunk],totalScore: chunk.score,diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsxindex f07000088..7c35b42d3 100644--- a/frontend/app/knowledge/page.tsx+++ b/frontend/app/knowledge/page.tsx@@ -60,6 +60,7 @@ function getSourceIcon(connectorType?: string) {return (<SharePointIcon className="h-4 w-4 text-foreground flex-shrink-0" />);+ case "openrag_docs":case "url":return <Globe className="h-4 w-4 text-muted-foreground flex-shrink-0" />;case "s3":@@ -104,7 +105,28 @@ function SearchPage() {} = useGetSearchQuery(queryOverride, parsedFilterData);const isOpenragDocsRow = useCallback((file?: File) => {- return file?.connector_type === "system_default";+ return (+ file?.connector_type === "openrag_docs" ||+ file?.connector_type === "system_default"+ );+ }, []);++ const getFileIdentity = useCallback((file?: File) => {+ if (!file) {+ return "";+ }++ const normalizedFilename = file.filename?.trim();+ if (normalizedFilename) {+ return normalizedFilename;+ }++ const normalizedSourceUrl = file.source_url?.trim();+ if (normalizedSourceUrl) {+ return normalizedSourceUrl;+ }++ return "";}, []);const hasOpenragRefreshCueFromTasks = tasks.some((task) => {@@ -147,10 +169,13 @@ function SearchPage() {}, [isError, error]);// Convert TaskFiles to File format and merge with backend resultsconst taskFilesAsFiles: File[] = taskFiles.map((taskFile) => {+ const normalizedFilename =+ taskFile.filename?.trim() || taskFile.source_url?.trim() || "Untitled source";+return {- filename: taskFile.filename,+ filename: normalizedFilename,mimetype: taskFile.mimetype,- source_url: taskFile.source_url,+ source_url: taskFile.source_url || "",size: taskFile.size,connector_type: taskFile.connector_type,status: taskFile.status,@@ -161,7 +186,7 @@ function SearchPage() {});// Create a map of task files by filename for quick lookupconst taskFileMap = new Map(- taskFilesAsFiles.map((file) => [file.filename, file]),+ taskFilesAsFiles.map((file) => [getFileIdentity(file), file]),);// Override backend files with task file status if they exist.// Keep openrag_docs rows sourced from indexed search results so@@ -170,7 +195,7 @@ function SearchPage() {if (file.connector_type === "openrag_docs") {return file;}- const taskFile = taskFileMap.get(file.filename);+ const taskFile = taskFileMap.get(getFileIdentity(file));if (taskFile) {// Override backend file with task file data (includes status)return { ...file, ...taskFile };@@ -195,15 +220,13 @@ function SearchPage() {return (taskFile.status !== "active" &&!backendFiles.some(- (backendFile) => backendFile.filename === taskFile.filename,+ (backendFile) => getFileIdentity(backendFile) === getFileIdentity(taskFile),));});// Combine task files first, then backend filesconst fileResults = [...backendFiles, ...filteredTaskFiles];- const hasOpenragDocsEntry = fileResults.some(- (file) => file.connector_type === "system_default",- );+ const hasOpenragDocsEntry = fileResults.some((file) => isOpenragDocsRow(file));const isDefaultKnowledgeView =!queryOverride || queryOverride.trim().length === 0 || queryOverride === "*";const shouldShowDefaultOpenragPlaceholder =@@ -217,7 +240,7 @@ function SearchPage() {mimetype: "text/markdown",source_url: "https://www.openr.ag/",size: 0,- connector_type: "system_default",+ connector_type: "openrag_docs",status: "unavailable",};const gridRows = shouldShowDefaultOpenragPlaceholder@@ -596,7 +619,7 @@ function SearchPage() {rowSelection="multiple"rowMultiSelectWithClick={false}suppressRowClickSelection={true}- getRowId={(params: GetRowIdParams<File>) => params.data?.filename}+ getRowId={(params: GetRowIdParams<File>) => getFileIdentity(params.data)}domLayout="normal"onSelectionChanged={onSelectionChanged}pagination={pagination}diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.pyindex cbcf6f0aa..c85e76323 100644--- a/src/services/langflow_file_service.py+++ b/src/services/langflow_file_service.py@@ -264,18 +264,18 @@ class LangflowFileService:config = get_openrag_config()embedding_model = config.knowledge.embedding_modelheaders = {- "X-Langflow-Global-Var-JWT": str(jwt_token) if jwt_token else "",- "X-Langflow-Global-Var-OWNER": str(owner) if owner else "",- "X-Langflow-Global-Var-OWNER_NAME": str(owner_name) if owner_name else "",- "X-Langflow-Global-Var-OWNER_EMAIL": str(owner_email) if owner_email else "",+ "X-Langflow-Global-Var-JWT": str(jwt_token),+ "X-Langflow-Global-Var-OWNER": str(owner),+ "X-Langflow-Global-Var-OWNER_NAME": str(owner_name),+ "X-Langflow-Global-Var-OWNER_EMAIL": str(owner_email),"X-Langflow-Global-Var-CONNECTOR_TYPE": str(connector_type),"X-Langflow-Global-Var-SELECTED_EMBEDDING_MODEL": str(embedding_model),- #TODO: Uncomment these when the flow is updated to use the new variables- # "X-Langflow-Global-Var-DOCS_URL": str(docs_url),- # "X-Langflow-Global-Var-DOCS_CRAWL_DEPTH": str(crawl_depth),- # "X-Langflow-Global-Var-DOCS_PREVENT_OUTSIDE": "true"- # if prevent_outside- # else "false",++ "X-Langflow-Global-Var-DOCUMENT_ID":"",+ "X-Langflow-Global-Var-SOURCE_URL": str(docs_url),++ "X-Langflow-Global-Var-ALLOWED_USERS": json.dumps( []),+ "X-Langflow-Global-Var-ALLOWED_GROUPS": json.dumps( []),}add_provider_credentials_to_headers(headers, config)@@ -295,6 +295,7 @@ class LangflowFileService:json=payload,headers=headers,)+ logger.info(f"[LF] Response for URL ingestion flow: {resp.json()}")if resp.status_code >= 400:logger.error("[LF] URL ingestion flow failed",