Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

Merge branch 'main' of github.com:langflow-ai/openrag into IBM_COS_S3_SPIKE

Mike Fortman091cfd13b7dd
diff --cc frontend/app/api/mutations/useRefreshOpenragDocs.ts
index 000000000,61ebbf553..5319428b6
mode 000000,100644..100644
--- a/frontend/app/api/mutations/useRefreshOpenragDocs.ts
+++ b/frontend/app/api/mutations/useRefreshOpenragDocs.ts
@@@ -1,0 -1,55 +1,53 @@@
+ "use client";
+
+ import { useMutation, useQueryClient } from "@tanstack/react-query";
+
+ interface RefreshOpenRAGDocsResponse {
+ message: string;
+ refreshed: boolean;
+ }
+
+ const refreshOpenragDocs = async (): Promise<RefreshOpenRAGDocsResponse> => {
+ const response = await fetch("/api/openrag-docs/refresh", {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ });
+
+ if (!response.ok) {
+ let errorMessage = "Failed to refresh OpenRAG docs";
+
+ try {
+ const contentType = response.headers.get("content-type") || "";
+ if (contentType.includes("application/json")) {
+ const error = await response.json();
+ errorMessage = error.detail || error.error || errorMessage;
+ } else {
+ const text = (await response.text()).trim();
+ if (text) {
+ errorMessage = text;
+ }
+ }
+ } catch {
+ // Keep default fallback message for malformed/non-JSON bodies.
+ }
+
- throw new Error(
- errorMessage,
- );
++ throw new Error(errorMessage);
+ }
+
+ return response.json();
+ };
+
+ export const useRefreshOpenragDocs = () => {
+ const queryClient = useQueryClient();
+
+ return useMutation({
+ mutationFn: refreshOpenragDocs,
+ onSettled: () => {
+ queryClient.invalidateQueries({ queryKey: ["tasks"], exact: false });
+ queryClient.invalidateQueries({ queryKey: ["search"], exact: false });
+ queryClient.invalidateQueries({ queryKey: ["settings"], exact: false });
+ },
+ });
+ };
diff --cc frontend/app/knowledge/page.tsx
index a17f95f8a,ea7d87605..2dc5e96af
--- a/frontend/app/knowledge/page.tsx
+++ b/frontend/app/knowledge/page.tsx
@@@ -1,5 -1,5 +1,6 @@@
"use client";
++import { useQueryClient } from "@tanstack/react-query";
import {
type CheckboxSelectionCallbackParams,
type ColDef,
@@@ -8,7 -8,7 +9,6 @@@
type ValueFormatterParams,
} from "ag-grid-community";
import { AgGridReact, type CustomCellRendererProps } from "ag-grid-react";
--import { useQueryClient } from "@tanstack/react-query";
import { Cloud, FileIcon, Globe, RefreshCw } from "lucide-react";
import { useRouter } from "next/navigation";
import { useCallback, useEffect, useRef, useState } from "react";
@@@ -40,13 -40,11 +40,14 @@@ import
DeleteConfirmationDialog,
formatFilesToDelete,
} from "../../components/delete-confirmation-dialog";
++import AwsLogo from "../../components/icons/aws-logo";
import GoogleDriveIcon from "../../components/icons/google-drive-logo";
- import IBMLogo from "../../components/icons/ibm-logo";
+import IBMCOSIcon from "../../components/icons/ibm-cos-icon";
++import IBMLogo from "../../components/icons/ibm-logo";
import OneDriveIcon from "../../components/icons/one-drive-logo";
import SharePointIcon from "../../components/icons/share-point-logo";
- import AwsLogo from "../../components/icons/aws-logo";
import { useDeleteDocument } from "../api/mutations/useDeleteDocument";
+ import { useRefreshOpenragDocs } from "../api/mutations/useRefreshOpenragDocs";
import { useSyncAllConnectors } from "../api/mutations/useSyncConnector";
// Function to get the appropriate icon for a connector type
@@@ -108,6 -104,51 +111,50 @@@ function SearchPage()
isError,
} = useGetSearchQuery(queryOverride, parsedFilterData);
+ const isOpenragDocsRow = useCallback((file?: File) => {
+ return (
+ file?.connector_type === "openrag_docs" ||
+ file?.connector_type === "system_default"
+ );
+ }, []);
+
+ const getFileIdentity = useCallback((file?: File) => {
+ if (!file) {
+ return "";
+ }
+
+ const normalizedFilename = file.filename?.trim();
+ if (normalizedFilename) {
+ return normalizedFilename;
+ }
+
+ const normalizedSourceUrl = file.source_url?.trim();
+ if (normalizedSourceUrl) {
+ return normalizedSourceUrl;
+ }
+
+ return "";
+ }, []);
+
+ const hasOpenragRefreshCueFromTasks = tasks.some((task) => {
+ const isTaskActive =
+ task.status === "pending" ||
+ task.status === "running" ||
+ task.status === "processing";
+ if (!isTaskActive || !task.files) {
+ return false;
+ }
+
+ return Object.entries(task.files).some(([fileKey, fileInfo]) => {
+ const filename = (fileInfo as { filename?: string })?.filename ?? "";
+ return (
- filename === "OpenRAG docs refresh" ||
- fileKey.includes("openr.ag")
++ filename === "OpenRAG docs refresh" || fileKey.includes("openr.ag")
+ );
+ });
+ });
+ const hasOpenragRefreshCue =
+ refreshOpenragDocsMutation.isPending || hasOpenragRefreshCueFromTasks;
+
// Show toast notification for search errors
useEffect(() => {
if (isError && error) {
@@@ -128,10 -169,13 +175,15 @@@
}, [isError, error]);
// Convert TaskFiles to File format and merge with backend results
const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => {
+ const normalizedFilename =
- taskFile.filename?.trim() || taskFile.source_url?.trim() || "Untitled source";
++ taskFile.filename?.trim() ||
++ taskFile.source_url?.trim() ||
++ "Untitled source";
+
return {
- filename: taskFile.filename,
+ filename: normalizedFilename,
mimetype: taskFile.mimetype,
- source_url: taskFile.source_url,
+ source_url: taskFile.source_url || "",
size: taskFile.size,
connector_type: taskFile.connector_type,
status: taskFile.status,
@@@ -158,12 -220,14 +228,15 @@@
return (
taskFile.status !== "active" &&
!backendFiles.some(
- (backendFile) => backendFile.filename === taskFile.filename,
- (backendFile) => getFileIdentity(backendFile) === getFileIdentity(taskFile),
++ (backendFile) =>
++ getFileIdentity(backendFile) === getFileIdentity(taskFile),
)
);
});
// Combine task files first, then backend files
const fileResults = [...backendFiles, ...filteredTaskFiles];
-
++
+ const gridRows = fileResults;
const gridRef = useRef<AgGridReact>(null);
const columnDefs: ColDef<File>[] = [
@@@ -276,6 -348,7 +357,8 @@@
headerName: "Status",
cellRenderer: ({ data }: CustomCellRendererProps<File>) => {
const status = data?.status || "active";
- const showOpenragRefreshCue = isOpenragDocsRow(data) && hasOpenragRefreshCue;
++ const showOpenragRefreshCue =
++ isOpenragDocsRow(data) && hasOpenragRefreshCue;
const error =
typeof data?.error === "string" && data.error.trim().length > 0
? data.error.trim()
@@@ -439,6 -545,35 +555,31 @@@
</>
)}
</Button>
+ <Button
+ type="button"
+ variant="outline"
+ className="rounded-lg flex-shrink-0"
+ disabled={refreshOpenragDocsMutation.isPending}
+ onClick={async () => {
+ try {
+ toast.info("Refreshing OpenRAG docs...");
+ const result = await refreshOpenragDocsMutation.mutateAsync();
+ toast.success(result.message);
+ } catch (error) {
+ toast.error(
+ error instanceof Error
+ ? error.message
+ : "Failed to refresh OpenRAG docs",
+ );
+ }
+ }}
+ >
+ {refreshOpenragDocsMutation.isPending ? (
- <>
- Refreshing docs...
- </>
++ <>Refreshing docs...</>
+ ) : (
- <>
- Fetch latest docs
- </>
++ <>Fetch latest docs</>
+ )}
+ </Button>
{selectedRows.length > 0 && (
<Button
type="button"
@@@ -464,7 -599,7 +605,9 @@@
rowSelection="multiple"
rowMultiSelectWithClick={false}
suppressRowClickSelection={true}
- getRowId={(params: GetRowIdParams<File>) => params.data?.filename}
- getRowId={(params: GetRowIdParams<File>) => getFileIdentity(params.data)}
++ getRowId={(params: GetRowIdParams<File>) =>
++ getFileIdentity(params.data)
++ }
domLayout="normal"
onSelectionChanged={onSelectionChanged}
pagination={pagination}
diff --cc frontend/components/knowledge-actions-dropdown.tsx
index 3e6bef307,693015169..c5edd6d80
--- a/frontend/components/knowledge-actions-dropdown.tsx
+++ b/frontend/components/knowledge-actions-dropdown.tsx
@@@ -19,8 -19,8 +19,8 @@@ import
TooltipProvider,
TooltipTrigger,
} from "@/components/ui/tooltip";
--import { formatFilesToDelete } from "@/lib/format-files-to-delete";
import { useTask } from "@/contexts/task-context";
++import { formatFilesToDelete } from "@/lib/format-files-to-delete";
import { DeleteConfirmationDialog } from "./delete-confirmation-dialog";
import { Button } from "./ui/button";
diff --cc src/main.py
index e48c41744,ea3a96045..31ecdf4f6
--- a/src/main.py
+++ b/src/main.py
@@@ -38,21 -44,16 +44,28 @@@ from api import
)
from api.connector_router import ConnectorRouter
+from connectors.ibm_cos.api import (
+ ibm_cos_defaults,
+ ibm_cos_configure,
+ ibm_cos_list_buckets,
+ ibm_cos_bucket_status,
+)
+from connectors.aws_s3.api import (
+ s3_defaults,
+ s3_configure,
+ s3_list_buckets,
+ s3_bucket_status,
+)
from services.api_key_service import APIKeyService
from api import keys as api_keys
- from api.v1 import chat as v1_chat, search as v1_search, documents as v1_documents, settings as v1_settings, models as v1_models, knowledge_filters as v1_knowledge_filters
+ from api.v1 import (
+ chat as v1_chat,
+ search as v1_search,
+ documents as v1_documents,
+ settings as v1_settings,
+ models as v1_models,
+ knowledge_filters as v1_knowledge_filters,
+ )
# Configuration and setup
from config.settings import (
@@@ -761,45 -1480,113 +1492,91 @@@ async def create_app()
# Authentication endpoints
app.add_api_route("/auth/init", auth.auth_init, methods=["POST"], tags=["internal"])
- app.add_api_route("/auth/callback", auth.auth_callback, methods=["POST"], tags=["internal"])
+ app.add_api_route(
+ "/auth/callback", auth.auth_callback, methods=["POST"], tags=["internal"]
+ )
app.add_api_route("/auth/me", auth.auth_me, methods=["GET"], tags=["internal"])
- app.add_api_route("/auth/logout", auth.auth_logout, methods=["POST"], tags=["internal"])
+ app.add_api_route(
+ "/auth/logout", auth.auth_logout, methods=["POST"], tags=["internal"]
+ )
# Connector endpoints
- app.add_api_route(
- "/connectors", connectors.list_connectors, methods=["GET"], tags=["internal"]
- )
- app.add_api_route(
- "/connectors/{connector_type}/sync",
- connectors.connector_sync,
- methods=["POST"],
- tags=["internal"],
- )
- app.add_api_route(
- "/connectors/sync-all",
- connectors.sync_all_connectors,
- methods=["POST"],
- tags=["internal"],
- )
- app.add_api_route(
- "/connectors/{connector_type}/status",
- connectors.connector_status,
- methods=["GET"],
- tags=["internal"],
- )
- app.add_api_route(
- "/connectors/{connector_type}/token",
- connectors.connector_token,
- methods=["GET"],
- tags=["internal"],
- )
- app.add_api_route(
- "/connectors/{connector_type}/disconnect",
- connectors.connector_disconnect,
- methods=["DELETE"],
- tags=["internal"],
- )
- app.add_api_route(
- "/connectors/{connector_type}/webhook",
- connectors.connector_webhook,
- methods=["POST", "GET"],
- tags=["internal"],
- )
+ app.add_api_route("/connectors", connectors.list_connectors, methods=["GET"], tags=["internal"])
+ # IBM COS-specific routes (registered before generic /{connector_type}/... to avoid shadowing)
+ app.add_api_route("/connectors/ibm_cos/defaults", ibm_cos_defaults, methods=["GET"], tags=["internal"])
+ app.add_api_route("/connectors/ibm_cos/configure", ibm_cos_configure, methods=["POST"], tags=["internal"])
+ app.add_api_route("/connectors/ibm_cos/{connection_id}/buckets", ibm_cos_list_buckets, methods=["GET"], tags=["internal"])
+ app.add_api_route("/connectors/ibm_cos/{connection_id}/bucket-status", ibm_cos_bucket_status, methods=["GET"], tags=["internal"])
+ # AWS S3-specific routes (registered before generic /{connector_type}/... to avoid shadowing)
+ app.add_api_route("/connectors/aws_s3/defaults", s3_defaults, methods=["GET"], tags=["internal"])
+ app.add_api_route("/connectors/aws_s3/configure", s3_configure, methods=["POST"], tags=["internal"])
+ app.add_api_route("/connectors/aws_s3/{connection_id}/buckets", s3_list_buckets, methods=["GET"], tags=["internal"])
+ app.add_api_route("/connectors/aws_s3/{connection_id}/bucket-status", s3_bucket_status, methods=["GET"], tags=["internal"])
+ app.add_api_route("/connectors/{connector_type}/sync", connectors.connector_sync, methods=["POST"], tags=["internal"])
+ app.add_api_route("/connectors/sync-all", connectors.sync_all_connectors, methods=["POST"], tags=["internal"])
+ app.add_api_route("/connectors/{connector_type}/status", connectors.connector_status, methods=["GET"], tags=["internal"])
+ app.add_api_route("/connectors/{connector_type}/token", connectors.connector_token, methods=["GET"], tags=["internal"])
+ app.add_api_route("/connectors/{connector_type}/disconnect", connectors.connector_disconnect, methods=["DELETE"], tags=["internal"])
+ app.add_api_route("/connectors/{connector_type}/webhook", connectors.connector_webhook, methods=["POST", "GET"], tags=["internal"])
# Document endpoints
- app.add_api_route("/documents/check-filename", documents.check_filename_exists, methods=["GET"], tags=["internal"])
- app.add_api_route("/documents/delete-by-filename", documents.delete_documents_by_filename, methods=["POST"], tags=["internal"])
+ app.add_api_route(
+ "/documents/check-filename",
+ documents.check_filename_exists,
+ methods=["GET"],
+ tags=["internal"],
+ )
+ app.add_api_route(
+ "/documents/delete-by-filename",
+ documents.delete_documents_by_filename,
+ methods=["POST"],
+ tags=["internal"],
+ )
# OIDC endpoints
- app.add_api_route("/.well-known/openid-configuration", oidc.oidc_discovery, methods=["GET"], tags=["internal"])
- app.add_api_route("/auth/jwks", oidc.jwks_endpoint, methods=["GET"], tags=["internal"])
- app.add_api_route("/auth/introspect", oidc.token_introspection, methods=["POST"], tags=["internal"])
+ app.add_api_route(
+ "/.well-known/openid-configuration",
+ oidc.oidc_discovery,
+ methods=["GET"],
+ tags=["internal"],
+ )
+ app.add_api_route(
+ "/auth/jwks", oidc.jwks_endpoint, methods=["GET"], tags=["internal"]
+ )
+ app.add_api_route(
+ "/auth/introspect",
+ oidc.token_introspection,
+ methods=["POST"],
+ tags=["internal"],
+ )
# Settings endpoints
- app.add_api_route("/settings", settings.get_settings, methods=["GET"], tags=["internal"])
- app.add_api_route("/settings", settings.update_settings, methods=["POST"], tags=["internal"])
- app.add_api_route("/onboarding/state", settings.update_onboarding_state, methods=["POST"], tags=["internal"])
+ app.add_api_route(
+ "/settings", settings.get_settings, methods=["GET"], tags=["internal"]
+ )
+ app.add_api_route(
+ "/settings", settings.update_settings, methods=["POST"], tags=["internal"]
+ )
+ app.add_api_route(
+ "/onboarding/state",
+ settings.update_onboarding_state,
+ methods=["POST"],
+ tags=["internal"],
+ )
+ app.add_api_route(
+ "/openrag-docs/refresh",
+ settings.refresh_openrag_docs,
+ methods=["POST"],
+ tags=["internal"],
+ )
# Provider health check endpoint
- app.add_api_route("/provider/health", provider_health.check_provider_health, methods=["GET"], tags=["internal"])
+ app.add_api_route(
+ "/provider/health",
+ provider_health.check_provider_health,
+ methods=["GET"],
+ tags=["internal"],
+ )
# Health check endpoints
app.add_api_route("/health", health_check, methods=["GET"], tags=["internal"])