OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
Merge branch 'main' of github.com:langflow-ai/openrag into IBM_COS_S3_SPIKE
Mike Fortman091cfd13b7dd
diff --cc frontend/app/api/mutations/useRefreshOpenragDocs.tsindex 000000000,61ebbf553..5319428b6mode 000000,100644..100644--- a/frontend/app/api/mutations/useRefreshOpenragDocs.ts+++ b/frontend/app/api/mutations/useRefreshOpenragDocs.ts@@@ -1,0 -1,55 +1,53 @@@+ "use client";++ import { useMutation, useQueryClient } from "@tanstack/react-query";++ interface RefreshOpenRAGDocsResponse {+ message: string;+ refreshed: boolean;+ }++ const refreshOpenragDocs = async (): Promise<RefreshOpenRAGDocsResponse> => {+ const response = await fetch("/api/openrag-docs/refresh", {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ },+ });++ if (!response.ok) {+ let errorMessage = "Failed to refresh OpenRAG docs";++ try {+ const contentType = response.headers.get("content-type") || "";+ if (contentType.includes("application/json")) {+ const error = await response.json();+ errorMessage = error.detail || error.error || errorMessage;+ } else {+ const text = (await response.text()).trim();+ if (text) {+ errorMessage = text;+ }+ }+ } catch {+ // Keep default fallback message for malformed/non-JSON bodies.+ }+- throw new Error(- errorMessage,- );++ throw new Error(errorMessage);+ }++ return response.json();+ };++ export const useRefreshOpenragDocs = () => {+ const queryClient = useQueryClient();++ return useMutation({+ mutationFn: refreshOpenragDocs,+ onSettled: () => {+ queryClient.invalidateQueries({ queryKey: ["tasks"], exact: false });+ queryClient.invalidateQueries({ queryKey: ["search"], exact: false });+ queryClient.invalidateQueries({ queryKey: ["settings"], exact: false });+ },+ });+ };diff --cc frontend/app/knowledge/page.tsxindex a17f95f8a,ea7d87605..2dc5e96af--- a/frontend/app/knowledge/page.tsx+++ b/frontend/app/knowledge/page.tsx@@@ -1,5 -1,5 +1,6 @@@"use client";++import { useQueryClient } from "@tanstack/react-query";import {type CheckboxSelectionCallbackParams,type ColDef,@@@ -8,7 -8,7 +9,6 @@@type ValueFormatterParams,} from "ag-grid-community";import { AgGridReact, type CustomCellRendererProps } from "ag-grid-react";--import { useQueryClient } from "@tanstack/react-query";import { Cloud, FileIcon, Globe, RefreshCw } from "lucide-react";import { useRouter } from "next/navigation";import { useCallback, useEffect, useRef, useState } from "react";@@@ -40,13 -40,11 +40,14 @@@ importDeleteConfirmationDialog,formatFilesToDelete,} from "../../components/delete-confirmation-dialog";++import AwsLogo from "../../components/icons/aws-logo";import GoogleDriveIcon from "../../components/icons/google-drive-logo";- import IBMLogo from "../../components/icons/ibm-logo";+import IBMCOSIcon from "../../components/icons/ibm-cos-icon";++import IBMLogo from "../../components/icons/ibm-logo";import OneDriveIcon from "../../components/icons/one-drive-logo";import SharePointIcon from "../../components/icons/share-point-logo";- import AwsLogo from "../../components/icons/aws-logo";import { useDeleteDocument } from "../api/mutations/useDeleteDocument";+ import { useRefreshOpenragDocs } from "../api/mutations/useRefreshOpenragDocs";import { useSyncAllConnectors } from "../api/mutations/useSyncConnector";// Function to get the appropriate icon for a connector type@@@ -108,6 -104,51 +111,50 @@@ function SearchPage()isError,} = useGetSearchQuery(queryOverride, parsedFilterData);+ const isOpenragDocsRow = useCallback((file?: File) => {+ return (+ file?.connector_type === "openrag_docs" ||+ file?.connector_type === "system_default"+ );+ }, []);++ const getFileIdentity = useCallback((file?: File) => {+ if (!file) {+ return "";+ }++ const normalizedFilename = file.filename?.trim();+ if (normalizedFilename) {+ return normalizedFilename;+ }++ const normalizedSourceUrl = file.source_url?.trim();+ if (normalizedSourceUrl) {+ return normalizedSourceUrl;+ }++ return "";+ }, []);++ const hasOpenragRefreshCueFromTasks = tasks.some((task) => {+ const isTaskActive =+ task.status === "pending" ||+ task.status === "running" ||+ task.status === "processing";+ if (!isTaskActive || !task.files) {+ return false;+ }++ return Object.entries(task.files).some(([fileKey, fileInfo]) => {+ const filename = (fileInfo as { filename?: string })?.filename ?? "";+ return (- filename === "OpenRAG docs refresh" ||- fileKey.includes("openr.ag")++ filename === "OpenRAG docs refresh" || fileKey.includes("openr.ag")+ );+ });+ });+ const hasOpenragRefreshCue =+ refreshOpenragDocsMutation.isPending || hasOpenragRefreshCueFromTasks;+// Show toast notification for search errorsuseEffect(() => {if (isError && error) {@@@ -128,10 -169,13 +175,15 @@@}, [isError, error]);// Convert TaskFiles to File format and merge with backend resultsconst taskFilesAsFiles: File[] = taskFiles.map((taskFile) => {+ const normalizedFilename =- taskFile.filename?.trim() || taskFile.source_url?.trim() || "Untitled source";++ taskFile.filename?.trim() ||++ taskFile.source_url?.trim() ||++ "Untitled source";+return {- filename: taskFile.filename,+ filename: normalizedFilename,mimetype: taskFile.mimetype,- source_url: taskFile.source_url,+ source_url: taskFile.source_url || "",size: taskFile.size,connector_type: taskFile.connector_type,status: taskFile.status,@@@ -158,12 -220,14 +228,15 @@@return (taskFile.status !== "active" &&!backendFiles.some(- (backendFile) => backendFile.filename === taskFile.filename,- (backendFile) => getFileIdentity(backendFile) === getFileIdentity(taskFile),++ (backendFile) =>++ getFileIdentity(backendFile) === getFileIdentity(taskFile),));});// Combine task files first, then backend filesconst fileResults = [...backendFiles, ...filteredTaskFiles];-+++ const gridRows = fileResults;const gridRef = useRef<AgGridReact>(null);const columnDefs: ColDef<File>[] = [@@@ -276,6 -348,7 +357,8 @@@headerName: "Status",cellRenderer: ({ data }: CustomCellRendererProps<File>) => {const status = data?.status || "active";- const showOpenragRefreshCue = isOpenragDocsRow(data) && hasOpenragRefreshCue;++ const showOpenragRefreshCue =++ isOpenragDocsRow(data) && hasOpenragRefreshCue;const error =typeof data?.error === "string" && data.error.trim().length > 0? data.error.trim()@@@ -439,6 -545,35 +555,31 @@@</>)}</Button>+ <Button+ type="button"+ variant="outline"+ className="rounded-lg flex-shrink-0"+ disabled={refreshOpenragDocsMutation.isPending}+ onClick={async () => {+ try {+ toast.info("Refreshing OpenRAG docs...");+ const result = await refreshOpenragDocsMutation.mutateAsync();+ toast.success(result.message);+ } catch (error) {+ toast.error(+ error instanceof Error+ ? error.message+ : "Failed to refresh OpenRAG docs",+ );+ }+ }}+ >+ {refreshOpenragDocsMutation.isPending ? (- <>- Refreshing docs...- </>++ <>Refreshing docs...</>+ ) : (- <>- Fetch latest docs- </>++ <>Fetch latest docs</>+ )}+ </Button>{selectedRows.length > 0 && (<Buttontype="button"@@@ -464,7 -599,7 +605,9 @@@rowSelection="multiple"rowMultiSelectWithClick={false}suppressRowClickSelection={true}- getRowId={(params: GetRowIdParams<File>) => params.data?.filename}- getRowId={(params: GetRowIdParams<File>) => getFileIdentity(params.data)}++ getRowId={(params: GetRowIdParams<File>) =>++ getFileIdentity(params.data)++ }domLayout="normal"onSelectionChanged={onSelectionChanged}pagination={pagination}diff --cc frontend/components/knowledge-actions-dropdown.tsxindex 3e6bef307,693015169..c5edd6d80--- a/frontend/components/knowledge-actions-dropdown.tsx+++ b/frontend/components/knowledge-actions-dropdown.tsx@@@ -19,8 -19,8 +19,8 @@@ importTooltipProvider,TooltipTrigger,} from "@/components/ui/tooltip";--import { formatFilesToDelete } from "@/lib/format-files-to-delete";import { useTask } from "@/contexts/task-context";++import { formatFilesToDelete } from "@/lib/format-files-to-delete";import { DeleteConfirmationDialog } from "./delete-confirmation-dialog";import { Button } from "./ui/button";diff --cc src/main.pyindex e48c41744,ea3a96045..31ecdf4f6--- a/src/main.py+++ b/src/main.py@@@ -38,21 -44,16 +44,28 @@@ from api import)from api.connector_router import ConnectorRouter+from connectors.ibm_cos.api import (+ ibm_cos_defaults,+ ibm_cos_configure,+ ibm_cos_list_buckets,+ ibm_cos_bucket_status,+)+from connectors.aws_s3.api import (+ s3_defaults,+ s3_configure,+ s3_list_buckets,+ s3_bucket_status,+)from services.api_key_service import APIKeyServicefrom api import keys as api_keys- from api.v1 import chat as v1_chat, search as v1_search, documents as v1_documents, settings as v1_settings, models as v1_models, knowledge_filters as v1_knowledge_filters+ from api.v1 import (+ chat as v1_chat,+ search as v1_search,+ documents as v1_documents,+ settings as v1_settings,+ models as v1_models,+ knowledge_filters as v1_knowledge_filters,+ )# Configuration and setupfrom config.settings import (@@@ -761,45 -1480,113 +1492,91 @@@ async def create_app()# Authentication endpointsapp.add_api_route("/auth/init", auth.auth_init, methods=["POST"], tags=["internal"])- app.add_api_route("/auth/callback", auth.auth_callback, methods=["POST"], tags=["internal"])+ app.add_api_route(+ "/auth/callback", auth.auth_callback, methods=["POST"], tags=["internal"]+ )app.add_api_route("/auth/me", auth.auth_me, methods=["GET"], tags=["internal"])- app.add_api_route("/auth/logout", auth.auth_logout, methods=["POST"], tags=["internal"])+ app.add_api_route(+ "/auth/logout", auth.auth_logout, methods=["POST"], tags=["internal"]+ )# Connector endpoints- app.add_api_route(- "/connectors", connectors.list_connectors, methods=["GET"], tags=["internal"]- )- app.add_api_route(- "/connectors/{connector_type}/sync",- connectors.connector_sync,- methods=["POST"],- tags=["internal"],- )- app.add_api_route(- "/connectors/sync-all",- connectors.sync_all_connectors,- methods=["POST"],- tags=["internal"],- )- app.add_api_route(- "/connectors/{connector_type}/status",- connectors.connector_status,- methods=["GET"],- tags=["internal"],- )- app.add_api_route(- "/connectors/{connector_type}/token",- connectors.connector_token,- methods=["GET"],- tags=["internal"],- )- app.add_api_route(- "/connectors/{connector_type}/disconnect",- connectors.connector_disconnect,- methods=["DELETE"],- tags=["internal"],- )- app.add_api_route(- "/connectors/{connector_type}/webhook",- connectors.connector_webhook,- methods=["POST", "GET"],- tags=["internal"],- )+ app.add_api_route("/connectors", connectors.list_connectors, methods=["GET"], tags=["internal"])+ # IBM COS-specific routes (registered before generic /{connector_type}/... to avoid shadowing)+ app.add_api_route("/connectors/ibm_cos/defaults", ibm_cos_defaults, methods=["GET"], tags=["internal"])+ app.add_api_route("/connectors/ibm_cos/configure", ibm_cos_configure, methods=["POST"], tags=["internal"])+ app.add_api_route("/connectors/ibm_cos/{connection_id}/buckets", ibm_cos_list_buckets, methods=["GET"], tags=["internal"])+ app.add_api_route("/connectors/ibm_cos/{connection_id}/bucket-status", ibm_cos_bucket_status, methods=["GET"], tags=["internal"])+ # AWS S3-specific routes (registered before generic /{connector_type}/... to avoid shadowing)+ app.add_api_route("/connectors/aws_s3/defaults", s3_defaults, methods=["GET"], tags=["internal"])+ app.add_api_route("/connectors/aws_s3/configure", s3_configure, methods=["POST"], tags=["internal"])+ app.add_api_route("/connectors/aws_s3/{connection_id}/buckets", s3_list_buckets, methods=["GET"], tags=["internal"])+ app.add_api_route("/connectors/aws_s3/{connection_id}/bucket-status", s3_bucket_status, methods=["GET"], tags=["internal"])+ app.add_api_route("/connectors/{connector_type}/sync", connectors.connector_sync, methods=["POST"], tags=["internal"])+ app.add_api_route("/connectors/sync-all", connectors.sync_all_connectors, methods=["POST"], tags=["internal"])+ app.add_api_route("/connectors/{connector_type}/status", connectors.connector_status, methods=["GET"], tags=["internal"])+ app.add_api_route("/connectors/{connector_type}/token", connectors.connector_token, methods=["GET"], tags=["internal"])+ app.add_api_route("/connectors/{connector_type}/disconnect", connectors.connector_disconnect, methods=["DELETE"], tags=["internal"])+ app.add_api_route("/connectors/{connector_type}/webhook", connectors.connector_webhook, methods=["POST", "GET"], tags=["internal"])# Document endpoints- app.add_api_route("/documents/check-filename", documents.check_filename_exists, methods=["GET"], tags=["internal"])- app.add_api_route("/documents/delete-by-filename", documents.delete_documents_by_filename, methods=["POST"], tags=["internal"])+ app.add_api_route(+ "/documents/check-filename",+ documents.check_filename_exists,+ methods=["GET"],+ tags=["internal"],+ )+ app.add_api_route(+ "/documents/delete-by-filename",+ documents.delete_documents_by_filename,+ methods=["POST"],+ tags=["internal"],+ )# OIDC endpoints- app.add_api_route("/.well-known/openid-configuration", oidc.oidc_discovery, methods=["GET"], tags=["internal"])- app.add_api_route("/auth/jwks", oidc.jwks_endpoint, methods=["GET"], tags=["internal"])- app.add_api_route("/auth/introspect", oidc.token_introspection, methods=["POST"], tags=["internal"])+ app.add_api_route(+ "/.well-known/openid-configuration",+ oidc.oidc_discovery,+ methods=["GET"],+ tags=["internal"],+ )+ app.add_api_route(+ "/auth/jwks", oidc.jwks_endpoint, methods=["GET"], tags=["internal"]+ )+ app.add_api_route(+ "/auth/introspect",+ oidc.token_introspection,+ methods=["POST"],+ tags=["internal"],+ )# Settings endpoints- app.add_api_route("/settings", settings.get_settings, methods=["GET"], tags=["internal"])- app.add_api_route("/settings", settings.update_settings, methods=["POST"], tags=["internal"])- app.add_api_route("/onboarding/state", settings.update_onboarding_state, methods=["POST"], tags=["internal"])+ app.add_api_route(+ "/settings", settings.get_settings, methods=["GET"], tags=["internal"]+ )+ app.add_api_route(+ "/settings", settings.update_settings, methods=["POST"], tags=["internal"]+ )+ app.add_api_route(+ "/onboarding/state",+ settings.update_onboarding_state,+ methods=["POST"],+ tags=["internal"],+ )+ app.add_api_route(+ "/openrag-docs/refresh",+ settings.refresh_openrag_docs,+ methods=["POST"],+ tags=["internal"],+ )# Provider health check endpoint- app.add_api_route("/provider/health", provider_health.check_provider_health, methods=["GET"], tags=["internal"])+ app.add_api_route(+ "/provider/health",+ provider_health.check_provider_health,+ methods=["GET"],+ tags=["internal"],+ )# Health check endpointsapp.add_api_route("/health", health_check, methods=["GET"], tags=["internal"])