Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

Normalize file identity and OpenRAG/Langflow changes

Edwin Jose33f9feb45f15
diff --git a/frontend/app/api/queries/useGetSearchQuery.ts b/frontend/app/api/queries/useGetSearchQuery.ts
index e0b96f719..88bb9d45e 100644
--- a/frontend/app/api/queries/useGetSearchQuery.ts
+++ b/frontend/app/api/queries/useGetSearchQuery.ts
@@ -71,6 +71,20 @@ export const useGetSearchQuery = (
) => {
const queryClient = useQueryClient();
+ const getFileIdentity = (chunk: ChunkResult): string => {
+ const normalizedFilename = chunk.filename?.trim();
+ if (normalizedFilename) {
+ return normalizedFilename;
+ }
+
+ const normalizedSourceUrl = chunk.source_url?.trim();
+ if (normalizedSourceUrl) {
+ return normalizedSourceUrl;
+ }
+
+ return "Untitled source";
+ };
+
// Normalize the query to match what will actually be searched
const effectiveQuery = query || queryData?.query || "*";
@@ -134,7 +148,8 @@ export const useGetSearchQuery = (
>();
(data.results || []).forEach((chunk: ChunkResult) => {
- const existing = fileMap.get(chunk.filename);
+ const fileIdentity = getFileIdentity(chunk);
+ const existing = fileMap.get(fileIdentity);
if (existing) {
existing.chunks.push(chunk);
existing.totalScore += chunk.score;
@@ -148,8 +163,8 @@ export const useGetSearchQuery = (
existing.embedding_dimensions = chunk.embedding_dimensions;
}
} else {
- fileMap.set(chunk.filename, {
- filename: chunk.filename,
+ fileMap.set(fileIdentity, {
+ filename: fileIdentity,
mimetype: chunk.mimetype,
chunks: [chunk],
totalScore: chunk.score,
diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsx
index f07000088..7c35b42d3 100644
--- a/frontend/app/knowledge/page.tsx
+++ b/frontend/app/knowledge/page.tsx
@@ -60,6 +60,7 @@ function getSourceIcon(connectorType?: string) {
return (
<SharePointIcon className="h-4 w-4 text-foreground flex-shrink-0" />
);
+ case "openrag_docs":
case "url":
return <Globe className="h-4 w-4 text-muted-foreground flex-shrink-0" />;
case "s3":
@@ -104,7 +105,28 @@ function SearchPage() {
} = useGetSearchQuery(queryOverride, parsedFilterData);
const isOpenragDocsRow = useCallback((file?: File) => {
- return file?.connector_type === "system_default";
+ return (
+ file?.connector_type === "openrag_docs" ||
+ file?.connector_type === "system_default"
+ );
+ }, []);
+
+ const getFileIdentity = useCallback((file?: File) => {
+ if (!file) {
+ return "";
+ }
+
+ const normalizedFilename = file.filename?.trim();
+ if (normalizedFilename) {
+ return normalizedFilename;
+ }
+
+ const normalizedSourceUrl = file.source_url?.trim();
+ if (normalizedSourceUrl) {
+ return normalizedSourceUrl;
+ }
+
+ return "";
}, []);
const hasOpenragRefreshCueFromTasks = tasks.some((task) => {
@@ -147,10 +169,13 @@ function SearchPage() {
}, [isError, error]);
// Convert TaskFiles to File format and merge with backend results
const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => {
+ const normalizedFilename =
+ taskFile.filename?.trim() || taskFile.source_url?.trim() || "Untitled source";
+
return {
- filename: taskFile.filename,
+ filename: normalizedFilename,
mimetype: taskFile.mimetype,
- source_url: taskFile.source_url,
+ source_url: taskFile.source_url || "",
size: taskFile.size,
connector_type: taskFile.connector_type,
status: taskFile.status,
@@ -161,7 +186,7 @@ function SearchPage() {
});
// Create a map of task files by filename for quick lookup
const taskFileMap = new Map(
- taskFilesAsFiles.map((file) => [file.filename, file]),
+ taskFilesAsFiles.map((file) => [getFileIdentity(file), file]),
);
// Override backend files with task file status if they exist.
// Keep openrag_docs rows sourced from indexed search results so
@@ -170,7 +195,7 @@ function SearchPage() {
if (file.connector_type === "openrag_docs") {
return file;
}
- const taskFile = taskFileMap.get(file.filename);
+ const taskFile = taskFileMap.get(getFileIdentity(file));
if (taskFile) {
// Override backend file with task file data (includes status)
return { ...file, ...taskFile };
@@ -195,15 +220,13 @@ function SearchPage() {
return (
taskFile.status !== "active" &&
!backendFiles.some(
- (backendFile) => backendFile.filename === taskFile.filename,
+ (backendFile) => getFileIdentity(backendFile) === getFileIdentity(taskFile),
)
);
});
// Combine task files first, then backend files
const fileResults = [...backendFiles, ...filteredTaskFiles];
- const hasOpenragDocsEntry = fileResults.some(
- (file) => file.connector_type === "system_default",
- );
+ const hasOpenragDocsEntry = fileResults.some((file) => isOpenragDocsRow(file));
const isDefaultKnowledgeView =
!queryOverride || queryOverride.trim().length === 0 || queryOverride === "*";
const shouldShowDefaultOpenragPlaceholder =
@@ -217,7 +240,7 @@ function SearchPage() {
mimetype: "text/markdown",
source_url: "https://www.openr.ag/",
size: 0,
- connector_type: "system_default",
+ connector_type: "openrag_docs",
status: "unavailable",
};
const gridRows = shouldShowDefaultOpenragPlaceholder
@@ -596,7 +619,7 @@ function SearchPage() {
rowSelection="multiple"
rowMultiSelectWithClick={false}
suppressRowClickSelection={true}
- getRowId={(params: GetRowIdParams<File>) => params.data?.filename}
+ getRowId={(params: GetRowIdParams<File>) => getFileIdentity(params.data)}
domLayout="normal"
onSelectionChanged={onSelectionChanged}
pagination={pagination}
diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.py
index cbcf6f0aa..c85e76323 100644
--- a/src/services/langflow_file_service.py
+++ b/src/services/langflow_file_service.py
@@ -264,18 +264,18 @@ class LangflowFileService:
config = get_openrag_config()
embedding_model = config.knowledge.embedding_model
headers = {
- "X-Langflow-Global-Var-JWT": str(jwt_token) if jwt_token else "",
- "X-Langflow-Global-Var-OWNER": str(owner) if owner else "",
- "X-Langflow-Global-Var-OWNER_NAME": str(owner_name) if owner_name else "",
- "X-Langflow-Global-Var-OWNER_EMAIL": str(owner_email) if owner_email else "",
+ "X-Langflow-Global-Var-JWT": str(jwt_token),
+ "X-Langflow-Global-Var-OWNER": str(owner),
+ "X-Langflow-Global-Var-OWNER_NAME": str(owner_name),
+ "X-Langflow-Global-Var-OWNER_EMAIL": str(owner_email),
"X-Langflow-Global-Var-CONNECTOR_TYPE": str(connector_type),
"X-Langflow-Global-Var-SELECTED_EMBEDDING_MODEL": str(embedding_model),
- #TODO: Uncomment these when the flow is updated to use the new variables
- # "X-Langflow-Global-Var-DOCS_URL": str(docs_url),
- # "X-Langflow-Global-Var-DOCS_CRAWL_DEPTH": str(crawl_depth),
- # "X-Langflow-Global-Var-DOCS_PREVENT_OUTSIDE": "true"
- # if prevent_outside
- # else "false",
+
+ "X-Langflow-Global-Var-DOCUMENT_ID":"",
+ "X-Langflow-Global-Var-SOURCE_URL": str(docs_url),
+
+ "X-Langflow-Global-Var-ALLOWED_USERS": json.dumps( []),
+ "X-Langflow-Global-Var-ALLOWED_GROUPS": json.dumps( []),
}
add_provider_credentials_to_headers(headers, config)
@@ -295,6 +295,7 @@ class LangflowFileService:
json=payload,
headers=headers,
)
+ logger.info(f"[LF] Response for URL ingestion flow: {resp.json()}")
if resp.status_code >= 400:
logger.error(
"[LF] URL ingestion flow failed",