OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: surface provider errors in chat instead of generic connection failures (#2127)
Wallgauac66bf0ecf52
diff --git a/Makefile b/Makefileindex b95465144..394319e32 100644--- a/Makefile+++ b/Makefile@@ -736,11 +736,11 @@ frontend: ## Run frontend locallyecho "$(YELLOW)Detected Power architecture ($$ARCH) — using Webpack fallback$(NC)"; \npx next dev --webpack \--port $$PORT \- --hostname $(hostname); \+ --hostname $$(hostname); \else \npx next dev \--port $$PORT \- --hostname $(hostname); \+ --hostname $$(hostname); \fidocling: ## Start docling-serve for document processingdiff --git a/frontend/app/chat/page.tsx b/frontend/app/chat/page.tsxindex 1e4f95195..759382c6b 100644--- a/frontend/app/chat/page.tsx+++ b/frontend/app/chat/page.tsx@@ -11,6 +11,11 @@ import { useTask } from "@/contexts/task-context";import { useOnboardingState } from "@/hooks/use-onboarding-state";import { useChatStreaming } from "@/hooks/useChatStreaming";import { trackLLMCall } from "@/lib/analytics";+import {+ dedupeConsecutiveErrorMessages,+ formatProviderErrorMessage,+ looksLikeProviderErrorContent,+} from "@/lib/chat-stream-errors";import { FILE_CONFIRMATION, FILES_REGEX } from "@/lib/constants";import { buildSearchPayloadFilters } from "@/lib/filter-normalization";import { uploadFileForContext } from "@/lib/upload-utils";@@ -76,6 +81,9 @@ function ChatPage() {const { scrollToBottom } = useStickToBottomContext();const lastLoadedConversationRef = useRef<string | null>(null);+ // Set when a live stream fails so history sync cannot replace one error card+ // with Langflow's duplicated copies of the same failure.+ const liveErrorConversationRef = useRef<string | null>(null);const { addTask } = useTask();// Check if chat history is loading@@ -111,17 +119,57 @@ function ChatPage() {} = useChatStreaming({endpoint: apiEndpoint,onComplete: (message, responseId) => {+ setLoading(false);+ setWaitingTooLong(false);++ setMessages((prev) => {+ if (!message.error) {+ return [...prev, message];+ }+ // One error card per failure — drop a trailing duplicate if present.+ const withoutTrailingDup = [...prev];+ while (+ withoutTrailingDup.length > 0 &&+ withoutTrailingDup[withoutTrailingDup.length - 1]?.role ===+ "assistant" &&+ withoutTrailingDup[withoutTrailingDup.length - 1]?.error &&+ withoutTrailingDup[withoutTrailingDup.length - 1]?.content ===+ message.content+ ) {+ withoutTrailingDup.pop();+ }+ return [...withoutTrailingDup, message];+ });++ if (message.error) {+ // Sidebar id stays on currentConversationId; onError clears Langflow chaining.+ if (responseId) {+ liveErrorConversationRef.current = responseId;+ if (!currentConversationId) {+ setCurrentConversationId(responseId);+ refreshConversations(true);+ if (conversationFilter && typeof window !== "undefined") {+ localStorage.setItem(+ `conversation_filter_${responseId}`,+ conversationFilter.id,+ );+ }+ } else {+ refreshConversationsSilent();+ }+ }+ return;+ }+trackLLMCall({mode: "chat",model: settings?.agent?.llm_model,inputTokens: message.usage?.input_tokens,outputTokens: message.usage?.output_tokens,});- setMessages((prev) => [...prev, message]);- setLoading(false);- setWaitingTooLong(false);if (responseId) {cancelNudges();+ // Langflow session id for chaining; sidebar id stays on currentConversationId.setPreviousResponseIds((prev) => ({...prev,[endpoint]: responseId,@@ -136,7 +184,8 @@ function ChatPage() {// Save filter association for this responseif (conversationFilter && typeof window !== "undefined") {- const newKey = `conversation_filter_${responseId}`;+ const stableId = currentConversationId || responseId;+ const newKey = `conversation_filter_${stableId}`;localStorage.setItem(newKey, conversationFilter.id);}}@@ -145,15 +194,13 @@ function ChatPage() {console.error("Streaming error:", error);setLoading(false);setWaitingTooLong(false);- // Set chat error flag to trigger test_completion=true on health checks+ // Set chat error flag to trigger test_completion=true on health checks.setChatError(true);- const errorMessage: Message = {- role: "assistant",- content:- "Sorry, I couldn't connect to the chat service. Please try again.",- timestamp: new Date(),- };- setMessages((prev) => [...prev, errorMessage]);+ // Clear Langflow session chaining; conversation_id keeps the sidebar thread.+ setPreviousResponseIds((prev) => ({+ ...prev,+ [endpoint]: null,+ }));},});@@ -269,6 +316,7 @@ function ChatPage() {setIsFilterHighlighted(false);setLoading(false);lastLoadedConversationRef.current = null;+ liveErrorConversationRef.current = null;// Focus input after a short delay to ensure rendering is completesetTimeout(() => {@@ -291,20 +339,47 @@ function ChatPage() {// Load conversation data from contextuseEffect(() => {let focusTimeoutId: NodeJS.Timeout;- // Only load conversation data when:- // 1. conversationData exists AND- // 2. (It's a different conversation OR we're not streaming and data has changed) AND- // 3. User is not in the middle of an interaction- const isNewConversation =- lastLoadedConversationRef.current !== conversationData?.response_id;- const hasMessageCountChanged =- conversationData?.messages?.length !== messages.length;-- if (- conversationData?.messages &&- (isNewConversation || (!isChatStreaming && hasMessageCountChanged)) &&+ // Only load conversation data when remote history should win:+ // - Switching to a different conversation always loads remote.+ // - Same conversation: never clobber local turns that are still ahead+ // (failed sends append user+error before history refreshes; syncing+ // stale remote would wipe them and retries then duplicate in Langflow).+ const conversationId = conversationData?.response_id ?? null;+ const isSwitchingConversation =+ conversationId != null &&+ lastLoadedConversationRef.current != null &&+ lastLoadedConversationRef.current !== conversationId;+ const isFirstLoadOfConversation =+ conversationId != null &&+ lastLoadedConversationRef.current !== conversationId;+ const remoteMessageCount = conversationData?.messages?.length ?? 0;+ const hasMessageCountChanged = remoteMessageCount !== messages.length;+ const localMessagesAhead = messages.length > remoteMessageCount;+ // After a live failed send we already appended the error locally. History+ // often returns the same provider failure repeated from Langflow — do not+ // clobber the live transcript for that conversation id.+ const skipSyncAfterLiveError =+ conversationId != null &&+ liveErrorConversationRef.current === conversationId &&+ !isSwitchingConversation;+ const shouldSyncSameConversation =+ !isChatStreaming &&+ hasMessageCountChanged &&+ !localMessagesAhead &&!isUserInteracting &&- !isForkingInProgress+ !isForkingInProgress;++ if (skipSyncAfterLiveError) {+ lastLoadedConversationRef.current = conversationId;+ setPreviousResponseIds((prev) => ({+ ...prev,+ [conversationData?.endpoint ?? endpoint]: null,+ }));+ } else if (+ conversationData?.messages &&+ (isSwitchingConversation ||+ (isFirstLoadOfConversation && !localMessagesAhead) ||+ (!isFirstLoadOfConversation && shouldSyncSameConversation))) {// Convert backend message format to frontend Message interfaceconst convertedMessages: Message[] = conversationData.messages.map(@@ -337,11 +412,17 @@ function ChatPage() {}>;response_data?: unknown;}) => {+ const isProviderError =+ Boolean(msg.error) ||+ (msg.role === "assistant" &&+ looksLikeProviderErrorContent(msg.content || ""));const message: Message = {role: msg.role as "user" | "assistant",- content: msg.content,+ content: isProviderError+ ? formatProviderErrorMessage(msg.content)+ : msg.content,timestamp: new Date(msg.timestamp || new Date()),- error: msg.error || false,+ error: isProviderError,};// Extract function calls from chunks or response_data@@ -458,13 +539,21 @@ function ChatPage() {},);- setMessages(convertedMessages);+ const dedupedMessages = dedupeConsecutiveErrorMessages(convertedMessages);+ setMessages(dedupedMessages);lastLoadedConversationRef.current = conversationData.response_id;+ if (liveErrorConversationRef.current === conversationData.response_id) {+ liveErrorConversationRef.current = null;+ }- // Set the previous response ID for this conversation+ // Don't chain a session that ended in an error — Langflow often collapses+ // follow-ups to "An unknown error occurred." and hides the real failure.+ const lastConverted = dedupedMessages[dedupedMessages.length - 1];setPreviousResponseIds((prev) => ({...prev,- [conversationData.endpoint]: conversationData.response_id,+ [conversationData.endpoint]: lastConverted?.error+ ? null+ : conversationData.response_id,}));// Focus input when loading a conversation@@ -484,6 +573,7 @@ function ChatPage() {setPreviousResponseIds,isChatStreaming,messages.length,+ endpoint,]);// Handle new conversation creation - only reset messages when placeholderConversation is set@@ -533,13 +623,21 @@ function ChatPage() {})(): undefined;- // Use passed previousResponseId if available, otherwise fall back to state- const responseIdToUse = previousResponseId || previousResponseIds[endpoint];+ // OpenRAG sidebar thread vs Langflow session are separate:+ // - conversationId keeps the same list entry after errors+ // - previousResponseId is omitted after an error so Langflow starts fresh+ const lastAssistant = [...messages]+ .reverse()+ .find((message) => message.role === "assistant");+ const langflowSessionId = lastAssistant?.error+ ? undefined+ : previousResponseId || previousResponseIds[endpoint] || undefined;// Use the hook to send the messageawait sendStreamingMessage({prompt: userMessage.content,- previousResponseId: responseIdToUse || undefined,+ previousResponseId: langflowSessionId,+ conversationId: currentConversationId || undefined,filters: processedFilters,filter_id: conversationFilter?.id, // ✅ Add filter_id for this conversationlimit: parsedFilterData?.limit ?? 10,diff --git a/frontend/app/onboarding/_components/onboarding-content.tsx b/frontend/app/onboarding/_components/onboarding-content.tsxindex 97f0a645d..7f0362f68 100644--- a/frontend/app/onboarding/_components/onboarding-content.tsx+++ b/frontend/app/onboarding/_components/onboarding-content.tsx@@ -148,13 +148,18 @@ export function OnboardingContent({const { streamingMessage, isLoading, sendMessage } = useChatStreaming({onComplete: async (message, newResponseId) => {+ setAssistantMessage(message);+ // Errors are display-only during onboarding — do not track or persist them.+ if (message.error) {+ return;+ }+trackLLMCall({mode: "onboarding",model: settings?.agent?.llm_model,inputTokens: message.usage?.input_tokens,outputTokens: message.usage?.output_tokens,});- setAssistantMessage(message);// Save assistant message to backendawait updateOnboardingMutation.mutateAsync({assistant_message: {@@ -194,11 +199,19 @@ export function OnboardingContent({},onError: (error) => {console.error("Chat error:", error);- setAssistantMessage({- role: "assistant",- content:- "Sorry, I couldn't connect to the chat service. Please try again.",- timestamp: new Date(),+ // Display is owned by onComplete (including message.error). Keep a+ // fallback only when completion never ran.+ setAssistantMessage((prev) => {+ if (prev?.error || (prev && !prev.isStreaming)) {+ return prev;+ }+ return {+ role: "assistant",+ content:+ error.message || "An error occurred while generating a response.",+ timestamp: new Date(),+ error: true,+ };});},});diff --git a/frontend/components/provider-health-banner.tsx b/frontend/components/provider-health-banner.tsxindex 722cb8ed9..faa345c10 100644--- a/frontend/components/provider-health-banner.tsx+++ b/frontend/components/provider-health-banner.tsx@@ -73,26 +73,27 @@ export function ProviderHealthBanner({ className }: ProviderHealthBannerProps) {let errorProvider: string | undefined;let errorMessage: string;+ // Prefer a single shared provider when LLM and embedding fail the same way+ // (e.g. both watsonx auth failures), so the banner stays readable.+ let showMultipleErrors = false;+if (llmError && embeddingError) {- // Both have errors - check if they're the sameif (llmError === embeddingError) {- // Same error for both - show onceerrorMessage = llmError;+ errorProvider =+ llmProvider === embeddingProvider ? llmProvider : undefined;} else {- // Different errors - show botherrorMessage = `${llmError}; ${embeddingError}`;+ errorProvider = undefined;+ showMultipleErrors = true;}- errorProvider = undefined; // Don't link to a specific provider} else if (llmError) {- // Only LLM has errorerrorProvider = llmProvider;errorMessage = llmError;} else if (embeddingError) {- // Only embedding has errorerrorProvider = embeddingProvider;errorMessage = embeddingError;} else {- // Fallback to original messageerrorMessage = health?.message || "Provider validation failed";errorProvider = llmProvider;}@@ -105,6 +106,10 @@ export function ProviderHealthBanner({ className }: ProviderHealthBannerProps) {? `/settings?setup=${errorProvider}`: "/settings";+ const bannerLabel = showMultipleErrors+ ? `Provider errors - ${errorMessage}`+ : `${providerTitle} error - ${errorMessage}`;+return (<BannerclassName={cn(@@ -117,13 +122,7 @@ export function ProviderHealthBanner({ className }: ProviderHealthBannerProps) {icon={AlertTriangle}/><BannerTitle className="font-medium flex items-center gap-2">- {llmError && embeddingError ? (- <>Provider errors - {errorMessage}</>- ) : (- <>- {providerTitle} error - {errorMessage}- </>- )}+ {bannerLabel}</BannerTitle><Button size="sm" onClick={() => router.push(settingsUrl)}>Fix Setupdiff --git a/frontend/contexts/task-context.tsx b/frontend/contexts/task-context.tsxindex 93ce5b78b..981f5f6f0 100644--- a/frontend/contexts/task-context.tsx+++ b/frontend/contexts/task-context.tsx@@ -28,6 +28,7 @@ import {getKnowledgeFileIdentity,inferTaskFileConnectorType,} from "@/lib/knowledge-table-state";+import { getTaskFailureToastDescription } from "@/lib/task-error-display";import {didTaskReachCompleted,didTaskReachTerminalState,@@ -496,9 +497,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {};if (isTotalFailure) {toast.error("Task failed", {- description: `${failedFiles} file${- failedFiles !== 1 ? "s" : ""- } failed`,+ description: getTaskFailureToastDescription(currentTask),action: toastAction,});} else {@@ -626,9 +625,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {resultValue: currentTask.error,});toast.error("Task failed", {- description: `Task ${currentTask.task_id} failed: ${- currentTask.error || "Unknown error"- }`,+ description: getTaskFailureToastDescription(currentTask),});// Set chat error flag to trigger test_completion=true on health checksdiff --git a/frontend/hooks/useChatStreaming.ts b/frontend/hooks/useChatStreaming.tsindex 2644846b3..761a23ce5 100644--- a/frontend/hooks/useChatStreaming.ts+++ b/frontend/hooks/useChatStreaming.ts@@ -5,6 +5,11 @@ import type {TokenUsage,} from "@/app/chat/_types/types";import { useChat } from "@/contexts/chat-context";+import {+ extractStreamProviderError,+ formatProviderErrorMessage,+ looksLikeProviderErrorContent,+} from "@/lib/chat-stream-errors";import {detectImplicitToolCall,detectRAGFromContent,@@ -24,6 +29,8 @@ interface UseChatStreamingOptions {interface SendMessageOptions {prompt: string;previousResponseId?: string;+ /** OpenRAG sidebar thread id (kept after errors; distinct from Langflow session). */+ conversationId?: string;filters?: FilterInput;filter_id?: string;limit?: number;@@ -47,6 +54,7 @@ export function useChatStreaming({const sendMessage = async ({prompt,previousResponseId,+ conversationId,filters,filter_id,limit = 10,@@ -55,6 +63,8 @@ export function useChatStreaming({// Set up timeout to detect stuck/hanging requestslet timeoutId: NodeJS.Timeout | null = null;let hasReceivedData = false;+ // Hoisted so the catch path can still attach a failed turn to history.+ let newResponseId: string | null = null;try {setIsLoading(true);@@ -86,6 +96,7 @@ export function useChatStreaming({prompt: string;stream: boolean;previous_response_id?: string;+ conversation_id?: string;filters?: FilterInput;filter_id?: string;limit?: number;@@ -101,6 +112,10 @@ export function useChatStreaming({requestBody.previous_response_id = previousResponseId;}+ if (conversationId) {+ requestBody.conversation_id = conversationId;+ }+if (filters) {const payloadFilters = buildSearchPayloadFilters(filters);if (payloadFilters) {@@ -139,9 +154,8 @@ export function useChatStreaming({let buffer = "";const content = { value: "" };const currentFunctionCalls: FunctionCall[] = [];- let newResponseId: string | null = null;- let isError = false;const usage: { value: TokenUsage | undefined } = { value: undefined };+ let providerStreamError: string | null = null;if (!controller.signal.aborted && thisStreamId === streamIdRef.current) {setStreamingMessage({@@ -190,12 +204,10 @@ export function useChatStreaming({parseOpenRAGChunk(chunk, content);detectImplicitToolCall(chunk, currentFunctionCalls);- if (- chunk.finish_reason === "error" ||- chunk.status === "failed"- ) {- console.error("Error detected in stream");- isError = true;+ const streamError = extractStreamProviderError(chunk);+ if (streamError) {+ console.error("Error detected in stream", streamError);+ providerStreamError = streamError;break streamLoop;}@@ -225,6 +237,21 @@ export function useChatStreaming({if (timeoutId) clearTimeout(timeoutId);}+ // Prefer accumulated stream text when it carries the real provider dump —+ // Langflow often sends finish_reason=error with an empty error payload.+ if (content.value && looksLikeProviderErrorContent(content.value)) {+ throw Object.assign(+ new Error(formatProviderErrorMessage(content.value)),+ { partialContent: content.value },+ );+ }++ if (providerStreamError) {+ throw Object.assign(new Error(providerStreamError), {+ partialContent: content.value,+ });+ }+if (!hasReceivedData ||(!content.value && currentFunctionCalls.length === 0)@@ -246,7 +273,6 @@ export function useChatStreaming({currentFunctionCalls.length > 0 ? currentFunctionCalls : undefined,timestamp: new Date(),isStreaming: false,- error: isError,usage: usage.value,};@@ -276,7 +302,9 @@ export function useChatStreaming({// Create user-friendly error messageconst errorMessage = (error as Error).message;- let errorContent = errorMessage; // Default to the actual error message+ let errorContent = formatProviderErrorMessage(+ errorMessage || "An error occurred while generating a response.",+ );// Only override with generic messages for specific infrastructure errorsif (errorMessage?.includes("timed out")) {@@ -291,9 +319,30 @@ export function useChatStreaming({errorContent ="Network error. Please check your connection and try again.";}- // For all other errors (including Langflow errors), use the actual error message- onError?.(error as Error);+ // Keep any mid-stream partial answer visible above the provider error,+ // unless the "partial" is itself a provider failure dump with JSON.+ const partialContent =+ typeof (error as { partialContent?: unknown }).partialContent ===+ "string"+ ? (error as { partialContent: string }).partialContent.trim()+ : "";+ const partialLooksLikeProviderError =+ partialContent.includes("{") ||+ /api key|authenticat|unauthorized|permission denied|rate limit/i.test(+ partialContent,+ );+ if (+ partialContent &&+ !partialLooksLikeProviderError &&+ !errorContent.startsWith(partialContent) &&+ !errorMessage?.includes("timed out") &&+ !errorMessage?.includes("No response") &&+ !errorMessage?.includes("NetworkError") &&+ !errorMessage?.includes("Failed to fetch")+ ) {+ errorContent = `${partialContent}\n\n${errorContent}`;+ }const errorMessageObj: Message = {role: "assistant",@@ -303,10 +352,19 @@ export function useChatStreaming({error: true,};- // Pass error message to onComplete so it gets added to chat history- // This ensures errors appear immediately and persist on page refresh- if (!streamAbortRef.current?.signal.aborted) {- onComplete?.(errorMessageObj, null);+ // onError owns side effects (flags, session reset). onComplete owns appending+ // the error message and refreshing history — pass any stream/store id so a+ // failed first turn still appears in the conversation list.+ onError?.(+ Object.assign(new Error(errorContent), {+ partialContent,+ }) as Error,+ );+ // User aborts skip completion; timeout aborts still surface the error card.+ const isTimeout = errorMessage?.includes("timed out");+ if (!streamAbortRef.current?.signal.aborted || isTimeout) {+ onComplete?.(errorMessageObj, newResponseId);+ refreshConversations(true);}return errorMessageObj;diff --git a/frontend/lib/chat-stream-errors.test.ts b/frontend/lib/chat-stream-errors.test.tsnew file mode 100644index 000000000..7a37dc758--- /dev/null+++ b/frontend/lib/chat-stream-errors.test.ts@@ -0,0 +1,175 @@+import assert from "node:assert/strict";+import { describe, it } from "node:test";+import {+ dedupeConsecutiveErrorMessages,+ extractStreamProviderError,+ formatProviderErrorMessage,+ looksLikeProviderErrorContent,+} from "./chat-stream-errors";++describe("extractStreamProviderError", () => {+ it("returns null for non-error chunks", () => {+ assert.equal(extractStreamProviderError({ status: "ok" }), null);+ assert.equal(extractStreamProviderError(null), null);+ });++ it("reads error.message from failed provider chunks", () => {+ assert.equal(+ extractStreamProviderError({+ status: "failed",+ finish_reason: "error",+ error: {+ message:+ "Rate limit exceeded for watsonx.ai. Please try again later.",+ },+ }),+ "Rate limit exceeded for watsonx.ai. Please try again later.",+ );+ });++ it("reads string error and top-level message fallbacks", () => {+ assert.equal(+ extractStreamProviderError({+ status: "failed",+ error: "Invalid API key for Anthropic.",+ }),+ "Invalid API key for Anthropic.",+ );+ assert.equal(+ extractStreamProviderError({+ finish_reason: "error",+ message: "Permission denied: model access not authorized.",+ }),+ "Permission denied: model access not authorized.",+ );+ });++ it("returns null when the error chunk has no message", () => {+ assert.equal(extractStreamProviderError({ status: "failed" }), null);+ assert.equal(extractStreamProviderError({ finish_reason: "error" }), null);+ });++ it("strips embedded JSON from provider error chunks", () => {+ assert.equal(+ extractStreamProviderError({+ status: "failed",+ error: {+ message:+ 'Failed to authenticate with IBM Watson: {"errorCode":"BXNIM0415E","errorMessage":"Provided API key could not be found."}',+ },+ }),+ "Provided API key could not be found.",+ );+ });+});++describe("looksLikeProviderErrorContent", () => {+ it("detects watsonx credential dumps streamed as assistant text", () => {+ assert.equal(+ looksLikeProviderErrorContent(+ 'Failed to initialize IBM WatsonX embedding model: Error: {"errorCode":"BXNIM0415E","errorMessage":"Provided API key could not be found."} An error occurred while generating a response.',+ ),+ true,+ );+ });++ it("detects permission-denied style provider errors", () => {+ assert.equal(+ looksLikeProviderErrorContent(+ "Permission denied: model access not authorized.",+ ),+ true,+ );+ });++ it("does not flag ordinary replies", () => {+ assert.equal(+ looksLikeProviderErrorContent("OpenRAG uses Langflow and OpenSearch."),+ false,+ );+ assert.equal(+ looksLikeProviderErrorContent(+ "The docs explain unauthorized access patterns and when permission denied responses appear.",+ ),+ false,+ );+ });++ it("flags IBM disabled API key messages", () => {+ assert.equal(+ looksLikeProviderErrorContent("Provided API key is disabled."),+ true,+ );+ });++ it("flags the exact generic stream fallback message", () => {+ assert.equal(+ looksLikeProviderErrorContent(+ "An error occurred while generating a response.",+ ),+ true,+ );+ });+});++describe("dedupeConsecutiveErrorMessages", () => {+ it("collapses repeated identical assistant errors", () => {+ const err = {+ role: "assistant",+ content: "Provided API key could not be found.",+ error: true,+ };+ assert.deepEqual(+ dedupeConsecutiveErrorMessages([+ { role: "user", content: "hello" },+ err,+ { ...err },+ { ...err },+ ]),+ [{ role: "user", content: "hello" }, err],+ );+ });+});++describe("formatProviderErrorMessage", () => {+ it("extracts OpenAI-style embedded JSON", () => {+ assert.equal(+ formatProviderErrorMessage(+ 'Provider request failed: {"error":{"message":"Incorrect API key provided","type":"invalid_request_error"}}',+ ),+ "Incorrect API key provided",+ );+ });++ it("strips embedded JSON even with trailing text", () => {+ assert.equal(+ formatProviderErrorMessage(+ 'Failed to authenticate. Error: {"errorCode":"BXNIM0415E","errorMessage":"Provided API key could not be found."} trailing junk',+ ),+ "Provided API key could not be found.",+ );+ });++ it("keeps a readable prefix when JSON is truncated", () => {+ assert.equal(+ formatProviderErrorMessage("Invalid API key {not-valid-json"),+ "Invalid API key",+ );+ });++ it("strips Error <label>: wrappers and prefers nested JSON messages", () => {+ assert.equal(+ formatProviderErrorMessage(+ "Error running graph: Error building Component Language Model: Rate limit exceeded",+ ),+ "Rate limit exceeded",+ );+ assert.equal(+ formatProviderErrorMessage(+ 'Error running graph: Error building Component Embedding Model: Failed to authenticate. Error: {"errorCode":"BXNIM0420E","errorMessage":"Provided API key is disabled."}',+ ),+ "Provided API key is disabled.",+ );+ assert.equal(formatProviderErrorMessage("Error: boom"), "Error: boom");+ });+});diff --git a/frontend/lib/chat-stream-errors.ts b/frontend/lib/chat-stream-errors.tsnew file mode 100644index 000000000..af2b5671c--- /dev/null+++ b/frontend/lib/chat-stream-errors.ts@@ -0,0 +1,286 @@+/**+ * Extract / sanitize provider stream errors for chat display.+ */++/**+ * Peel leading "Error <label>:" wrappers (Langflow graph/component noise, etc.).+ * Requires a label after "Error" so bare "Error: …" messages are preserved.+ */+const ERROR_LABEL_PREFIX = /^Error\s+\S[^:]*:\s*/i;++function stripErrorLabelPrefixes(text: string): string {+ let cleaned = text.trim();+ while (true) {+ const updated = cleaned.replace(ERROR_LABEL_PREFIX, "").trim();+ if (updated === cleaned) {+ return cleaned;+ }+ cleaned = updated;+ }+}++function tryParseJsonMessage(text: string): string | null {+ try {+ const data = JSON.parse(text) as Record<string, unknown>;+ if (!data || typeof data !== "object") {+ return null;+ }++ const errors = data.errors;+ if (Array.isArray(errors) && errors[0] && typeof errors[0] === "object") {+ const first = errors[0] as { message?: unknown; code?: unknown };+ if (typeof first.message === "string" && first.message.trim()) {+ return first.message;+ }+ if (typeof first.code === "string" && first.code.trim()) {+ return `Error: ${first.code}`;+ }+ }++ const errorObj = data.error;+ if (errorObj && typeof errorObj === "object") {+ const message = (errorObj as { message?: unknown }).message;+ if (typeof message === "string" && message.trim()) {+ return message;+ }+ }++ if (typeof data.message === "string" && data.message.trim()) {+ return data.message;+ }+ if (typeof data.errorMessage === "string" && data.errorMessage.trim()) {+ return data.errorMessage;+ }+ if (typeof data.detail === "string" && data.detail.trim()) {+ return data.detail;+ }+ } catch {+ return null;+ }+ return null;+}++function extractBalancedJsonObject(text: string): string | null {+ const start = text.indexOf("{");+ if (start < 0) {+ return null;+ }+ let depth = 0;+ let inString = false;+ let escaped = false;+ for (let i = start; i < text.length; i++) {+ const ch = text[i];+ if (inString) {+ if (escaped) {+ escaped = false;+ } else if (ch === "\\") {+ escaped = true;+ } else if (ch === '"') {+ inString = false;+ }+ continue;+ }+ if (ch === '"') {+ inString = true;+ } else if (ch === "{") {+ depth += 1;+ } else if (ch === "}") {+ depth -= 1;+ if (depth === 0) {+ return text.slice(start, i + 1);+ }+ }+ }+ return null;+}++/**+ * True when assistant text looks like a provider/auth failure rather than a reply.+ * Marker list aligned with src/api/provider_validation.py (_PROVIDER_ERROR_CONTENT_MARKERS).+ * Ambiguous phrases use tighter patterns so ordinary assistant prose is not flagged.+ */+export function looksLikeProviderErrorContent(text: string): boolean {+ const trimmed = text.trim();+ if (!trimmed) {+ return false;+ }+ if (trimmed.startsWith("Error:")) {+ return true;+ }+ const lowered = trimmed.toLowerCase().replace(/\.$/, "");+ if (+ lowered === "an unknown error occurred" ||+ lowered === "an error occurred while generating a response" ||+ lowered === "unknown error occurred" ||+ lowered === "something went wrong"+ ) {+ return true;+ }+ const markers = [+ "incorrect api key",+ "invalid api key",+ "invalid_api_key",+ "api key could not be found",+ "api key is invalid",+ "api key has been revoked",+ "api key revoked",+ "revoked api key",+ "provided api key could not be found",+ // IBM IAM: key still exists but is toggled off (vs deleted/not found).+ "api key is disabled",+ "api key disabled",+ "authentication_error",+ "failed to authenticate",+ "invalid x-api-key",+ "authentication failed",+ "invalid credentials",+ "could not authenticate",+ "rate limit",+ "rate_limit",+ "provider request failed",+ "insufficient_quota",+ ];+ if (markers.some((marker) => lowered.includes(marker))) {+ return true;+ }+ // Tighter matches for phrases that also appear in ordinary explanations.+ if (+ /(?:^|\berror\b[:\s]*|\b(?:401|403)\b[:\s]*)unauthorized\b/.test(lowered) ||+ /(?:^|\berror\b[:\s]*)permission denied\b/.test(lowered) ||+ /\bpermission denied\s*:/.test(lowered) ||+ /(?:^|\berror\b[:\s]*)quota exceeded\b/.test(lowered) ||+ /\bquota exceeded\s*:/.test(lowered)+ ) {+ return true;+ }+ if (+ trimmed.includes("{") &&+ (lowered.includes('"error"') ||+ lowered.includes('"errormessage"') ||+ lowered.includes('"errorcode"') ||+ lowered.includes('"errors"'))+ ) {+ return true;+ }+ return false;+}++/**+ * Strip embedded provider JSON payloads so chat never shows raw error objects.+ */+export function formatProviderErrorMessage(text: string): string {+ const trimmed = stripErrorLabelPrefixes(text.trim());+ if (!trimmed) {+ return "An error occurred while generating a response.";+ }++ const direct = tryParseJsonMessage(trimmed);+ if (direct) {+ // Pure JSON payloads: prefer the extracted message only.+ return stripErrorLabelPrefixes(direct);+ }++ const jsonBlob = extractBalancedJsonObject(trimmed);+ if (jsonBlob) {+ const nested = tryParseJsonMessage(jsonBlob);+ if (nested) {+ // Embedded provider JSON already has the real message — drop wrappers.+ return stripErrorLabelPrefixes(nested);+ }+ const prefix = stripErrorLabelPrefixes(+ trimmed+ .slice(0, trimmed.indexOf(jsonBlob))+ .replace(/[:\s]+$/, "")+ .trim(),+ );+ if (prefix) {+ return prefix;+ }+ }++ // Truncated / unbalanced JSON — keep the readable prefix only.+ const jsonStart = trimmed.indexOf("{");+ if (jsonStart > 0) {+ const prefix = stripErrorLabelPrefixes(+ trimmed+ .slice(0, jsonStart)+ .replace(/[:\s]+$/, "")+ .trim(),+ );+ if (prefix) {+ return prefix;+ }+ }++ return trimmed;+}++/**+ * Collapse consecutive identical assistant error messages.+ * Langflow often stores the same provider failure multiple times in one session.+ */+export function dedupeConsecutiveErrorMessages<+ T extends { role?: string; content?: string; error?: boolean },+>(messages: T[]): T[] {+ const result: T[] = [];+ for (const message of messages) {+ const prev = result[result.length - 1];+ const isErrorAssistant =+ message.role === "assistant" &&+ (Boolean(message.error) ||+ looksLikeProviderErrorContent(message.content || ""));+ const prevIsSameError =+ prev?.role === "assistant" &&+ (Boolean(prev.error) ||+ looksLikeProviderErrorContent(prev.content || "")) &&+ (prev.content || "").trim() === (message.content || "").trim();+ if (isErrorAssistant && prevIsSameError) {+ continue;+ }+ result.push(message);+ }+ return result;+}++/**+ * Extract a user-facing provider/stream error message from an NDJSON chunk.+ * Returns null when the chunk is not a failed/error terminal event.+ */+export function extractStreamProviderError(chunk: unknown): string | null {+ if (!chunk || typeof chunk !== "object") {+ return null;+ }++ const c = chunk as {+ finish_reason?: unknown;+ status?: unknown;+ error?: unknown;+ message?: unknown;+ };++ if (c.finish_reason !== "error" && c.status !== "failed") {+ return null;+ }++ let raw: string | null = null;++ if (typeof c.error === "string" && c.error.trim()) {+ raw = c.error;+ } else if (c.error && typeof c.error === "object") {+ const message = (c.error as { message?: unknown }).message;+ if (typeof message === "string" && message.trim()) {+ raw = message;+ }+ } else if (typeof c.message === "string" && c.message.trim()) {+ raw = c.message;+ }++ // Empty error payloads are common from Langflow (finish_reason=error with no+ // message). Return null so the client can fall back to accumulated content or+ // a later sanitized error chunk instead of a generic placeholder.+ if (!raw) {+ return null;+ }++ return formatProviderErrorMessage(raw);+}diff --git a/frontend/lib/task-error-display.ts b/frontend/lib/task-error-display.tsindex 87aa39406..b6b4c9a49 100644--- a/frontend/lib/task-error-display.ts+++ b/frontend/lib/task-error-display.ts@@ -173,6 +173,37 @@ export function resolveTaskFileError(return "Unknown error";}+/** Best available user-facing reason for a failed ingestion task toast. */+export function getTaskFailureToastDescription(task: {+ error?: string | null;+ failed_files?: number;+ files?: Record<string, TaskFileEntry> | null;+}): string {+ const files = Object.values(task.files ?? {});+ for (const fileInfo of files) {+ if (fileInfo.status !== "failed" && fileInfo.status !== "error") {+ continue;+ }+ const message = resolveTaskFileError(fileInfo, task.error ?? undefined);+ if (message && message !== "Unknown error") {+ return message;+ }+ }++ if (typeof task.error === "string" && task.error.trim()) {+ return task.error.trim();+ }++ const failedCount =+ task.failed_files ??+ files.filter((file) => file.status === "failed" || file.status === "error")+ .length;+ if (failedCount > 0) {+ return `${failedCount} file${failedCount !== 1 ? "s" : ""} failed`;+ }+ return "Unknown error";+}+export function analyzeTaskFileIngestionFailure(fileInfo: TaskFileEntry,taskError?: string,diff --git a/frontend/tests/core/tasks-unified-panel.spec.ts b/frontend/tests/core/tasks-unified-panel.spec.tsindex 3d6797a29..bf2d2f902 100644--- a/frontend/tests/core/tasks-unified-panel.spec.ts+++ b/frontend/tests/core/tasks-unified-panel.spec.ts@@ -193,8 +193,9 @@ test("completed task with failures keeps failure log in Tasks panel", async ({page,"Synthetic ingestion failure for test",);+ // Prefer title= on the panel error row — toast description can repeat the same text.await expect(- page.getByText("Synthetic ingestion failure for test"),+ page.getByTitle("Synthetic ingestion failure for test"),).toBeVisible();});@@ -242,7 +243,8 @@ test("completed task with failures requires View click to open tasks panel", asyawait openTasksPanel(page);await openPastTasksSection(page);await expandFirstFailureAccordion(page, "Auto-open on partial success");- await expect(page.getByText("Auto-open on partial success")).toBeVisible();+ // Prefer title= on the panel error row — toast description can repeat the same text.+ await expect(page.getByTitle("Auto-open on partial success")).toBeVisible();});test("new failed task auto-opens tasks panel", async ({ page }) => {@@ -287,7 +289,8 @@ test("new failed task auto-opens tasks panel", async ({ page }) => {await openTasksPanel(page);await openPastTasksSection(page);await expandFirstFailureAccordion(page, "Auto-open on failed task");- await expect(page.getByText("Auto-open on failed task")).toBeVisible();+ // Prefer title= on the panel error row — toast description can repeat the same text.+ await expect(page.getByTitle("Auto-open on failed task")).toBeVisible();});test("unified panel shows all completed tasks in a single past tasks section", async ({diff --git a/src/agent.py b/src/agent.pyindex a9ee54a0b..bfeec9114 100644--- a/src/agent.py+++ b/src/agent.py@@ -1,3 +1,4 @@+import uuidfrom typing import Anyfrom services.conversation_persistence_service import conversation_persistence@@ -9,6 +10,109 @@ logger = get_logger(__name__)active_conversations: dict[str, dict[str, Any]] = {}+def _format_provider_error_message(exc: BaseException) -> str:+ """Return a concise, user-facing provider error string from an exception."""+ from api.provider_validation import sanitize_provider_error_content++ return sanitize_provider_error_content(str(exc))+++def _provider_error_display_text(error_text: str, partial_response: str = "") -> str:+ """Combine any streamed partial answer with the provider error for display/history.++ If the mid-stream text is itself a provider failure dump (often raw JSON), keep+ only the sanitized error — do not paste the dump above the clean message.+ """+ from api.provider_validation import looks_like_provider_error_content++ partial = (partial_response or "").strip()+ if not partial:+ return error_text+ if looks_like_provider_error_content(partial) or "{" in partial:+ return error_text+ if error_text in partial:+ return partial+ return f"{partial}\n\n{error_text}"+++def _conversation_thread_in_memory(user_id: str, response_id: str | None) -> bool:+ """True when response_id is already keyed in the in-process conversation cache."""+ return bool(+ response_id+ and user_id in active_conversations+ and response_id in active_conversations[user_id]+ )+++def _resolve_conversation_store_id(+ response_id: str | None,+ thread_id: str | None,+ *,+ previous_loaded_from_memory: bool,+ mint_if_missing: bool = False,+) -> str | None:+ """Resolve the OpenRAG conversation id used for persistence.++ Prefer an in-memory thread id so follow-ups stay on one sidebar entry.+ Never reuse a cold thread id (would overwrite metadata with empty state).+ Fall back to the stream id, then optionally mint a UUID for failed first turns.+ """+ if previous_loaded_from_memory and thread_id:+ return thread_id+ if response_id:+ return response_id+ if mint_if_missing:+ return str(uuid.uuid4())+ return None+++def _conversation_ended_with_error(user_id: str, response_id: str | None) -> bool:+ """True when the in-memory thread's last assistant message is a provider error."""+ if not response_id or user_id not in active_conversations:+ return False+ messages = active_conversations[user_id].get(response_id, {}).get("messages") or []+ for message in reversed(messages):+ if message.get("role") == "assistant":+ return bool(message.get("error"))+ return False+++async def _persist_error_conversation(+ user_id: str, store_id: str, conversation_state: dict+) -> None:+ """Store a failed turn and claim session ownership.++ Claiming is required: otherwise the client may retry with this response_id and+ ``_assert_owns`` returns 404 (session_not_found) for an unclaimed session.+ """+ from datetime import datetime++ from services.session_ownership_service import session_ownership_service++ conversation_state["last_activity"] = datetime.now()+ await store_conversation_thread(user_id, store_id, conversation_state)+ try:+ await session_ownership_service.claim_session(user_id, store_id)+ logger.debug(f"Claimed session {store_id} for user {user_id} after stream error")+ except Exception as e:+ logger.warning(f"Failed to claim session ownership after stream error: {e}")+++def _stream_error_chunk(message: str, response_id: str | None = None) -> bytes:+ """NDJSON error event consumed by the chat streaming client."""+ import json++ payload: dict[str, Any] = {+ "status": "failed",+ "finish_reason": "error",+ "error": {"message": message},+ }+ if response_id:+ payload["id"] = response_id+ payload["response_id"] = response_id+ return (json.dumps(payload) + "\n").encode("utf-8")++async def get_user_conversations(user_id: str):"""Get conversation metadata for a user from persistent storage."""return await conversation_persistence.get_user_conversations(user_id)@@ -60,6 +164,7 @@ async def store_conversation_thread(user_id: str, response_id: str, conversationcontent = first_user_msg.get("content", "")title = content[:50] + "..." if len(content) > 50 else content+ user_assistant_messages = [msg for msg in messages if msg.get("role") in ["user", "assistant"]]metadata_only = {"response_id": response_id,"title": title,@@ -68,11 +173,19 @@ async def store_conversation_thread(user_id: str, response_id: str, conversation"last_activity": conversation_state.get("last_activity"),"previous_response_id": conversation_state.get("previous_response_id"),"filter_id": conversation_state.get("filter_id"),- "total_messages": len(- [msg for msg in messages if msg.get("role") in ["user", "assistant"]]- ),- # Don't store actual messages - Langflow has them+ "total_messages": len(user_assistant_messages),}+ # Snapshot error-thread turns: Langflow history is incomplete after fresh-session retries.+ if any(msg.get("error") for msg in user_assistant_messages):+ metadata_only["messages"] = [+ {+ "role": msg.get("role"),+ "content": msg.get("content"),+ "timestamp": msg.get("timestamp"),+ "error": bool(msg.get("error")),+ }+ for msg in user_assistant_messages+ ]await conversation_persistence.store_conversation_thread(user_id, response_id, metadata_only)@@ -145,13 +258,16 @@ async def async_response_stream(if hasattr(chunk, "output_text") and chunk.output_text:full_response += chunk.output_textelif hasattr(chunk, "delta") and chunk.delta:- # Handle delta properly - it might be a dict or string+ # Handle delta properly - it might be a dict or string.+ # Do not fall back to str(dict) when content is "" — that pollutes+ # the transcript with "{'content': ''}" prefixes on error chunks.if isinstance(chunk.delta, dict):- delta_text = (- chunk.delta.get("content", "")- or chunk.delta.get("text", "")- or str(chunk.delta)- )+ raw_delta = chunk.delta.get("content")+ if raw_delta is None:+ raw_delta = chunk.delta.get("text")+ delta_text = raw_delta if isinstance(raw_delta, str) else ""+ elif chunk.delta is None:+ delta_text = ""else:delta_text = str(chunk.delta)full_response += delta_text@@ -489,6 +605,7 @@ async def async_chat_stream(previous_response_id: str = None,filter_id: str = None,):+ previous_loaded_from_memory = _conversation_thread_in_memory(user_id, previous_response_id)# Get the specific conversation thread (or create new one)conversation_state = get_conversation_thread(user_id, previous_response_id)@@ -505,52 +622,76 @@ async def async_chat_stream(full_response = ""response_id = Noneusage_data = None- async for chunk in async_stream(- async_client,- prompt,- model,- previous_response_id=previous_response_id,- log_prefix="agent",- ):- # Extract text content to build full response for history- try:- import json+ try:+ async for chunk in async_stream(+ async_client,+ prompt,+ model,+ previous_response_id=previous_response_id,+ log_prefix="agent",+ ):+ # Extract text content to build full response for history+ try:+ import json- chunk_data = json.loads(chunk.decode("utf-8"))- if "delta" in chunk_data and "content" in chunk_data["delta"]:- full_response += chunk_data["delta"]["content"]- # Extract response_id from chunk- if "id" in chunk_data:- response_id = chunk_data["id"]- elif "response_id" in chunk_data:- response_id = chunk_data["response_id"]- # Capture usage from response.completed event- if chunk_data.get("type") == "response.completed":- response_obj = chunk_data.get("response", {})- usage_data = response_obj.get("usage")- except Exception:- pass- yield chunk+ chunk_data = json.loads(chunk.decode("utf-8"))+ if "delta" in chunk_data and "content" in chunk_data["delta"]:+ full_response += chunk_data["delta"]["content"]+ # Extract response_id from chunk+ if "id" in chunk_data:+ response_id = chunk_data["id"]+ elif "response_id" in chunk_data:+ response_id = chunk_data["response_id"]+ # Capture usage from response.completed event+ if chunk_data.get("type") == "response.completed":+ response_obj = chunk_data.get("response", {})+ usage_data = response_obj.get("usage")+ except Exception:+ pass+ yield chunk- # Add the complete assistant response to message history with response_id and timestamp- assistant_message = {- "role": "assistant",- "content": full_response,- "response_id": response_id,- "timestamp": datetime.now(),- }- # Store usage data if available (from response.completed event)- if usage_data:- assistant_message["response_data"] = {"usage": usage_data}- conversation_state["messages"].append(assistant_message)+ # Add the complete assistant response to message history with response_id and timestamp+ assistant_message = {+ "role": "assistant",+ "content": full_response,+ "response_id": response_id,+ "timestamp": datetime.now(),+ }+ # Store usage data if available (from response.completed event)+ if usage_data:+ assistant_message["response_data"] = {"usage": usage_data}+ conversation_state["messages"].append(assistant_message)- # Store the conversation thread with its response_id- if response_id:- conversation_state["last_activity"] = datetime.now()- await store_conversation_thread(user_id, response_id, conversation_state)- logger.debug(- f"Stored conversation thread for user {user_id} with response_id: {response_id}"+ # Store the conversation thread with its response_id+ if response_id:+ conversation_state["last_activity"] = datetime.now()+ await store_conversation_thread(user_id, response_id, conversation_state)+ logger.debug(+ f"Stored conversation thread for user {user_id} with response_id: {response_id}"+ )+ except Exception as e:+ logger.error(f"Error in chat stream: {e}", exc_info=True)+ error_text = _format_provider_error_message(e)+ display_text = _provider_error_display_text(error_text, full_response)+ conversation_state["messages"].append(+ {+ "role": "assistant",+ "content": display_text,+ "timestamp": datetime.now(),+ "error": True,+ }+ )+ store_id = _resolve_conversation_store_id(+ response_id,+ previous_response_id,+ previous_loaded_from_memory=previous_loaded_from_memory,+ mint_if_missing=True,)+ try:+ await _persist_error_conversation(user_id, store_id, conversation_state)+ except Exception as store_error:+ logger.error(f"Failed to store error conversation: {store_error}")+ yield _stream_error_chunk(display_text, store_id)# Async langflow function with conversation storage (non-streaming)@@ -561,6 +702,7 @@ async def async_langflow_chat(user_id: str,extra_headers: dict = None,previous_response_id: str = None,+ conversation_id: str = None,store_conversation: bool = True,filter_id: str = None,):@@ -568,11 +710,13 @@ async def async_langflow_chat("async_langflow_chat called",user_id=user_id,previous_response_id=previous_response_id,+ conversation_id=conversation_id,)+ thread_id = conversation_id or previous_response_idif store_conversation:# Get the specific conversation thread (or create new one)- conversation_state = get_conversation_thread(user_id, previous_response_id)+ conversation_state = get_conversation_thread(user_id, thread_id)logger.debug("Got langflow conversation state",message_count=len(conversation_state["messages"]),@@ -730,23 +874,11 @@ async def async_langflow_chat(except Exception as e:logger.warning(f"Failed to claim session ownership: {e}")- logger.debug(- f"Stored langflow conversation thread for user {user_id} with response_id: {response_id}"- )logger.debug("Stored langflow conversation thread",user_id=user_id,response_id=response_id,)-- # Debug: Check what's in user_conversations now- conversations = await get_user_conversations(user_id)- logger.debug(- "User conversations updated",- user_id=user_id,- conversation_count=len(conversations),- conversation_ids=list(conversations.keys()),- )else:logger.warning("No response_id received from langflow, conversation not stored")@@ -761,27 +893,35 @@ async def async_langflow_chat_stream(user_id: str,extra_headers: dict = None,previous_response_id: str = None,+ conversation_id: str = None,filter_id: str = None,):logger.debug("async_langflow_chat_stream called",user_id=user_id,previous_response_id=previous_response_id,+ conversation_id=conversation_id,)- # Get the specific conversation thread (or create new one)- conversation_state = get_conversation_thread(user_id, previous_response_id)+ # conversation_id keeps the OpenRAG sidebar thread; previous_response_id chains+ # the Langflow session. After errors the client omits previous_response_id.+ thread_id = conversation_id or previous_response_id+ previous_loaded_from_memory = _conversation_thread_in_memory(user_id, thread_id)+ conversation_state = get_conversation_thread(user_id, thread_id)- # Add user message to conversation with timestampfrom datetime import datetime- user_message = {"role": "user", "content": prompt, "timestamp": datetime.now()}- conversation_state["messages"].append(user_message)-- # Store filter_id in conversation state if provided+ conversation_state["messages"].append(+ {"role": "user", "content": prompt, "timestamp": datetime.now()}+ )if filter_id:conversation_state["filter_id"] = filter_id+ # Safety net: never continue a Langflow session that already ended in error.+ langflow_previous_response_id = previous_response_id+ if previous_response_id and _conversation_ended_with_error(user_id, thread_id):+ langflow_previous_response_id = None+full_response = ""response_id = Noneusage_data = None@@ -794,7 +934,7 @@ async def async_langflow_chat_stream(prompt,flow_id,extra_headers=extra_headers,- previous_response_id=previous_response_id,+ previous_response_id=langflow_previous_response_id,log_prefix="langflow",):# Extract text content to build full response for history@@ -812,13 +952,32 @@ async def async_langflow_chat_stream(elif "response_id" in chunk_data:response_id = chunk_data["response_id"]- # Check for error status+ # Check for error status. Do not forward bare Langflow error+ # chunks — they often lack a useful message and the client would+ # show a generic failure before our sanitized terminal chunk.if (chunk_data.get("finish_reason") == "error"or chunk_data.get("status") == "failed"):error_occurred = Truelogger.error("Error detected in Langflow stream chunk")+ err = chunk_data.get("error")+ if isinstance(err, dict):+ err_msg = err.get("message")+ else:+ err_msg = err+ if isinstance(err_msg, str) and err_msg.strip():+ if err_msg not in full_response:+ full_response = (+ f"{full_response}\n\n{err_msg}" if full_response else err_msg+ )+ elif (+ isinstance(chunk_data.get("message"), str) and chunk_data["message"].strip()+ ):+ msg = chunk_data["message"]+ if msg not in full_response:+ full_response = f"{full_response}\n\n{msg}" if full_response else msg+ continue# Capture usage from response.completed eventif chunk_data.get("type") == "response.completed":response_obj = chunk_data.get("response", {})@@ -827,7 +986,39 @@ async def async_langflow_chat_stream(passyield chunk- # Add the complete assistant response to message history with response_id, timestamp, and function call data+ from api.provider_validation import (+ looks_like_provider_error_content,+ resolve_chat_stream_error_message,+ )++ # Credential failures are often streamed as plain assistant text with no+ # exception. Sanitize, mark as error, and emit a terminal error chunk so+ # the client shows the error card instead of raw JSON.+ if error_occurred or looks_like_provider_error_content(full_response):+ display_text = resolve_chat_stream_error_message(full_response)+ conversation_state["messages"].append(+ {+ "role": "assistant",+ "content": display_text,+ "response_id": response_id,+ "timestamp": datetime.now(),+ "chunks": collected_chunks,+ "error": True,+ }+ )+ store_id = _resolve_conversation_store_id(+ response_id,+ thread_id,+ previous_loaded_from_memory=previous_loaded_from_memory,+ mint_if_missing=True,+ )+ try:+ await _persist_error_conversation(user_id, store_id, conversation_state)+ except Exception as store_error:+ logger.error(f"Failed to store error conversation: {store_error}")+ yield _stream_error_chunk(display_text, store_id)+ return+if full_response:assistant_message = {"role": "assistant",@@ -841,50 +1032,48 @@ async def async_langflow_chat_stream(assistant_message["response_data"] = {"usage": usage_data}conversation_state["messages"].append(assistant_message)- # Store the conversation thread with its response_id- if response_id:+ persist_id = _resolve_conversation_store_id(+ response_id,+ thread_id,+ previous_loaded_from_memory=previous_loaded_from_memory,+ )+ if persist_id:conversation_state["last_activity"] = datetime.now()- await store_conversation_thread(user_id, response_id, conversation_state)-+ await store_conversation_thread(user_id, persist_id, conversation_state)try:from services.session_ownership_service import session_ownership_service- await session_ownership_service.claim_session(user_id, response_id)- logger.debug(f"Claimed session {response_id} for user {user_id}")+ await session_ownership_service.claim_session(user_id, persist_id)except Exception as e:logger.warning(f"Failed to claim session ownership: {e}")-- logger.debug(- f"Stored langflow conversation thread for user {user_id} with response_id: {response_id}"- )except Exception as e:- # Log the errorlogger.error(f"Error in langflow chat stream: {e}", exc_info=True)- error_occurred = True+ from api.provider_validation import resolve_chat_stream_error_message- # Store error message in conversation history so it persists- error_message = {- "role": "assistant",- "content": f"Sorry, I encountered an error: {str(e)}",- "timestamp": datetime.now(),- "error": True,- }- conversation_state["messages"].append(error_message)-- # Try to store the conversation with error message- # Use a temporary response_id if we don't have one- if not response_id:- response_id = f"error_{user_id}_{int(datetime.now().timestamp())}"+ error_text = resolve_chat_stream_error_message(e)+ display_text = _provider_error_display_text(error_text, full_response)+ if full_response:+ display_text = resolve_chat_stream_error_message(display_text)+ conversation_state["messages"].append(+ {+ "role": "assistant",+ "content": display_text,+ "timestamp": datetime.now(),+ "error": True,+ }+ )+ store_id = _resolve_conversation_store_id(+ response_id,+ thread_id,+ previous_loaded_from_memory=previous_loaded_from_memory,+ mint_if_missing=True,+ )try:- conversation_state["last_activity"] = datetime.now()- await store_conversation_thread(user_id, response_id, conversation_state)- logger.debug(f"Stored conversation with error for user {user_id}")+ await _persist_error_conversation(user_id, store_id, conversation_state)except Exception as store_error:logger.error(f"Failed to store error conversation: {store_error}")-- # Re-raise the exception so it propagates to the API layer- raise+ yield _stream_error_chunk(display_text, store_id)async def delete_user_conversation(user_id: str, response_id: str) -> bool:diff --git a/src/api/chat.py b/src/api/chat.pyindex 23e8b5eb3..a3baa7e1c 100644--- a/src/api/chat.py+++ b/src/api/chat.py@@ -1,17 +1,16 @@-from typing import Optional, Any, Dict+from typing import Anyfrom fastapi import Depends, HTTPException-from pydantic import BaseModelfrom fastapi.responses import JSONResponse, StreamingResponse-from utils.logging_config import get_logger+from pydantic import BaseModelfrom dependencies import (get_chat_service,get_session_manager,- get_current_user,require_permission,)from session_manager import User+from utils.logging_config import get_loggerlogger = get_logger(__name__)@@ -20,7 +19,7 @@ def _openrag_user_id(user: User) -> str:return getattr(user, "db_user_id", None) or user.user_id-async def _assert_owns(session_id: Optional[str], user_id: str) -> None:+async def _assert_owns(session_id: str | None, user_id: str) -> None:"""Raise 403 if `session_id` is set but not owned by `user_id`.No-op when `session_id` is None (new conversation, nothing to check).@@ -30,6 +29,7 @@ async def _assert_owns(session_id: Optional[str], user_id: str) -> None:if not session_id:returnfrom services.session_ownership_service import session_ownership_service+owner = await session_ownership_service.get_session_owner(session_id)if owner is None:raise HTTPException(status_code=404, detail={"error": "session_not_found"})@@ -39,12 +39,15 @@ async def _assert_owns(session_id: Optional[str], user_id: str) -> None:class ChatBody(BaseModel):prompt: str- previous_response_id: Optional[str] = None+ previous_response_id: str | None = None+ # OpenRAG sidebar/thread id. Distinct from previous_response_id so a retry+ # after an error can stay in the same chat while starting a fresh Langflow session.+ conversation_id: str | None = Nonestream: bool = False- filters: Optional[Dict[str, Any]] = None+ filters: dict[str, Any] | None = Nonelimit: int = 10scoreThreshold: float = 0- filter_id: Optional[str] = None+ filter_id: str | None = Noneasync def chat_endpoint(@@ -64,9 +67,11 @@ async def chat_endpoint(if body.filters:from auth_context import set_search_filters+set_search_filters(body.filters)- from auth_context import set_search_limit, set_score_threshold+ from auth_context import set_score_threshold, set_search_limit+set_search_limit(body.limit)set_score_threshold(body.scoreThreshold)@@ -114,14 +119,17 @@ async def langflow_endpoint(storage_user_id = _openrag_user_id(user)await _assert_owns(body.previous_response_id, storage_user_id)+ await _assert_owns(body.conversation_id, storage_user_id)jwt_token = user.jwt_tokenif body.filters:from auth_context import set_search_filters+set_search_filters(body.filters)- from auth_context import set_search_limit, set_score_threshold+ from auth_context import set_score_threshold, set_search_limit+set_search_limit(body.limit)set_score_threshold(body.scoreThreshold)@@ -133,6 +141,7 @@ async def langflow_endpoint(user.user_id,jwt_token,previous_response_id=body.previous_response_id,+ conversation_id=body.conversation_id,stream=True,filter_id=body.filter_id,owner=user.user_id,@@ -154,6 +163,7 @@ async def langflow_endpoint(user.user_id,jwt_token,previous_response_id=body.previous_response_id,+ conversation_id=body.conversation_id,stream=False,filter_id=body.filter_id,owner=user.user_id,@@ -163,11 +173,9 @@ async def langflow_endpoint()return JSONResponse(result)- except Exception as e:+ except Exception:logger.exception("[CHAT] Langflow request failed")- return JSONResponse(- {"error": f"Langflow request failed: {str(e)}"}, status_code=500- )+ return JSONResponse({"error": "Langflow request failed"}, status_code=500)async def chat_history_endpoint(@@ -178,11 +186,9 @@ async def chat_history_endpoint(try:history = await chat_service.get_chat_history(_openrag_user_id(user))return JSONResponse(history)- except Exception as e:+ except Exception:logger.exception("[CHAT] Failed to get chat history")- return JSONResponse(- {"error": f"Failed to get chat history: {str(e)}"}, status_code=500- )+ return JSONResponse({"error": "Failed to get chat history"}, status_code=500)async def langflow_history_endpoint(@@ -193,11 +199,9 @@ async def langflow_history_endpoint(try:history = await chat_service.get_langflow_history(_openrag_user_id(user))return JSONResponse(history)- except Exception as e:+ except Exception:logger.exception("[CHAT] Failed to get langflow history")- return JSONResponse(- {"error": f"Failed to get langflow history: {str(e)}"}, status_code=500- )+ return JSONResponse({"error": "Failed to get langflow history"}, status_code=500)async def delete_session_endpoint(@@ -215,11 +219,8 @@ async def delete_session_endpoint(return JSONResponse({"message": "Session deleted successfully"})else:return JSONResponse(- {"error": result.get("error", "Failed to delete session")},- status_code=500+ {"error": result.get("error", "Failed to delete session")}, status_code=500)- except Exception as e:- logger.error(f"Error deleting session: {e}")- return JSONResponse(- {"error": f"Failed to delete session: {str(e)}"}, status_code=500- )+ except Exception:+ logger.exception("Error deleting session")+ return JSONResponse({"error": "Failed to delete session"}, status_code=500)diff --git a/src/api/provider_health.py b/src/api/provider_health.pyindex adb27636a..bc5aab9a7 100644--- a/src/api/provider_health.py+++ b/src/api/provider_health.py@@ -6,7 +6,7 @@ import httpxfrom fastapi import Dependsfrom fastapi.responses import JSONResponse-from api.provider_validation import validate_provider_setup+from api.provider_validation import format_provider_error_message, validate_provider_setupfrom config.settings import get_openrag_configfrom dependencies import require_permissionfrom session_manager import User@@ -195,10 +195,10 @@ async def check_provider_health(llm_error = None # Don't treat as errorlogger.info(f"LLM provider ({provider}) appears busy: {str(e)}")else:- llm_error = str(e)+ llm_error = format_provider_error_message(e)logger.error(f"LLM provider ({provider}) validation timed out: {llm_error}")except Exception as e:- llm_error = str(e)+ llm_error = format_provider_error_message(e)logger.error(f"LLM provider ({provider}) validation failed: {llm_error}")# Validate embedding provider@@ -229,12 +229,12 @@ async def check_provider_health(embedding_error = None # Don't treat as errorlogger.info(f"Embedding provider ({embedding_provider}) appears busy: {str(e)}")else:- embedding_error = str(e)+ embedding_error = format_provider_error_message(e)logger.error(f"Embedding provider ({embedding_provider}) validation timed out: {embedding_error}")except Exception as e:- embedding_error = str(e)+ embedding_error = format_provider_error_message(e)logger.error(f"Embedding provider ({embedding_provider}) validation failed: {embedding_error}")@@ -283,7 +283,7 @@ async def check_provider_health(except Exception as e:if _health_leader_key:provider_health_cache.release_error(_health_leader_key)- error_message = str(e)+ error_message = format_provider_error_message(e)logger.error(f"Provider health check failed for {provider}: {error_message}")return JSONResponse(diff --git a/src/api/provider_validation.py b/src/api/provider_validation.pyindex 431663210..494a80098 100644--- a/src/api/provider_validation.py+++ b/src/api/provider_validation.py@@ -1,17 +1,336 @@"""Provider validation utilities for testing API keys and models during onboarding."""import json+import re+import httpx+from utils.container_utils import transform_localhost_urlfrom utils.logging_config import get_loggerlogger = get_logger(__name__)++# Peel leading "Error <label>:" wrappers (e.g. Langflow graph/component noise).+# Requires a label after "Error" so bare "Error: …" messages are preserved.+_ERROR_LABEL_PREFIX_RE = re.compile(r"^Error\s+\S[^:]*:\s*", re.IGNORECASE)+++def _strip_error_label_prefixes(text: str) -> str:+ """Remove leading ``Error <label>:`` segments; keep the innermost message."""+ cleaned = text.strip()+ while True:+ updated = _ERROR_LABEL_PREFIX_RE.sub("", cleaned, count=1).strip()+ if updated == cleaned:+ return cleaned+ cleaned = updated+++_PROVIDER_CREDENTIAL_ERROR_MARKERS = (+ "incorrect api key",+ "invalid api key",+ "invalid_api_key",+ "api key could not be found",+ "api key is invalid",+ "api key has been revoked",+ "api key revoked",+ "revoked api key",+ "provided api key could not be found",+ # IBM IAM returns this when the key still exists but is toggled off.+ "api key is disabled",+ "api key disabled",+ "authentication_error",+ "failed to authenticate",+ "invalid x-api-key",+ "unauthorized",+ "authentication failed",+ "invalid credentials",+ "could not authenticate",+)+++_PROVIDER_ERROR_CONTENT_MARKERS = _PROVIDER_CREDENTIAL_ERROR_MARKERS + (+ "rate limit",+ "rate_limit",+ "permission denied",+ "quota exceeded",+ "provider request failed",+ "insufficient_quota",+)+++def _extract_balanced_json_object(text: str) -> str | None:+ """Return the first top-level `{...}` substring, ignoring trailing junk."""+ start = text.find("{")+ if start < 0:+ return None+ depth = 0+ in_string = False+ escape = False+ for i in range(start, len(text)):+ ch = text[i]+ if in_string:+ if escape:+ escape = False+ elif ch == "\\":+ escape = True+ elif ch == '"':+ in_string = False+ continue+ if ch == '"':+ in_string = True+ elif ch == "{":+ depth += 1+ elif ch == "}":+ depth -= 1+ if depth == 0:+ return text[start : i + 1]+ return None+++def format_provider_error_message(exc: BaseException | str) -> str:+ """Return a concise, user-facing provider error string from an exception or text."""+ text = _strip_error_label_prefixes((str(exc) if not isinstance(exc, str) else exc).strip())+ if not text:+ return "An error occurred while generating a response."++ parsed = _parse_json_error_message(text)+ if parsed != text:+ # Pure JSON (or JSON-shaped) payloads: prefer the extracted message only.+ return _strip_error_label_prefixes(parsed)++ json_blob = _extract_balanced_json_object(text)+ if json_blob:+ nested = _parse_json_error_message(json_blob)+ if nested != json_blob:+ # Embedded provider JSON already has the real message — drop wrappers.+ return _strip_error_label_prefixes(nested)++ return text+++def is_provider_credential_error(text: str | BaseException | None) -> bool:+ """True when the error indicates an invalid, missing, or revoked provider API key."""+ if text is None:+ return False+ lowered = (str(text) if not isinstance(text, str) else text).lower()+ return any(marker in lowered for marker in _PROVIDER_CREDENTIAL_ERROR_MARKERS)+++_GENERIC_UPSTREAM_ERROR_MARKERS = (+ "an unknown error occurred",+ "an error occurred while generating a response",+ "unknown error occurred",+ "something went wrong",+)+++def is_generic_upstream_error(text: str | BaseException | None) -> bool:+ """True when upstream returned an opaque error with no actionable detail."""+ if text is None:+ return True+ lowered = (str(text) if not isinstance(text, str) else text).strip().lower().rstrip(".")+ if not lowered:+ return True+ generics = {marker.rstrip(".") for marker in _GENERIC_UPSTREAM_ERROR_MARKERS}+ return lowered in generics+++def looks_like_provider_error_content(text: str | None) -> bool:+ """True when assistant text looks like a provider/auth failure rather than a normal reply."""+ if not text or not text.strip():+ return False+ content = text.strip()+ if content.startswith("Error:"):+ return True+ if is_generic_upstream_error(content):+ return True+ lowered = content.lower()+ if any(marker in lowered for marker in _PROVIDER_ERROR_CONTENT_MARKERS):+ return True+ # Raw provider payloads often embed JSON error objects.+ if "{" in content and (+ '"error"' in lowered+ or '"errormessage"' in lowered+ or '"errorcode"' in lowered+ or '"errors"' in lowered+ ):+ return True+ return False+++def sanitize_provider_error_content(text: str | BaseException | None) -> str:+ """Format provider error text and fall back when JSON still leaks through."""+ if text is None:+ return "An error occurred while generating a response."+ if not isinstance(text, str):+ text = str(text)+ if not text.strip():+ return "An error occurred while generating a response."+ cleaned = format_provider_error_message(text)+ if "{" in cleaned or "}" in cleaned:+ if is_provider_credential_error(text) or is_provider_credential_error(cleaned):+ return (+ "The configured API key is invalid or has been revoked. "+ "Update it in Settings and retry."+ )+ # Strip the first JSON object as a last resort.+ json_start = cleaned.find("{")+ prefix = cleaned[:json_start].rstrip(": ").strip()+ return prefix or "An error occurred while generating a response."+ return cleaned+++_LANGFLOW_TRANSPORT_ERROR_MARKERS = (+ "server disconnected",+ "disconnected without sending a response",+ "remote protocol error",+ "connection reset",+ "broken pipe",+ "all connection attempts failed",+ "connection refused",+)+++def is_langflow_transport_error(text: str | BaseException | None) -> bool:+ """True when Langflow dropped the HTTP connection mid-ingest."""+ if text is None:+ return False+ lowered = (str(text) if not isinstance(text, str) else text).lower()+ return any(marker in lowered for marker in _LANGFLOW_TRANSPORT_ERROR_MARKERS)+++async def _probe_provider_credential_error(+ *,+ provider: str | None,+ api_key: str | None,+ endpoint: str | None = None,+ project_id: str | None = None,+ embedding_model: str | None = None,+ llm_model: str | None = None,+) -> str | None:+ """Run a lightweight provider check; return a cleaned credential error if auth fails."""+ if not provider:+ return None+ try:+ await validate_provider_setup(+ provider=provider,+ api_key=api_key,+ embedding_model=embedding_model,+ llm_model=llm_model,+ endpoint=endpoint,+ project_id=project_id,+ test_completion=False,+ )+ except Exception as probe_exc:+ cleaned = sanitize_provider_error_content(probe_exc)+ if is_provider_credential_error(probe_exc) or is_provider_credential_error(cleaned):+ return cleaned+ return None+++async def probe_provider_credential_error() -> str | None:+ """Return a credential error if any ingest-relevant provider key fails auth.++ Langflow receives every configured provider key, so a revoked watsonx key can+ crash ingest even when the selected embedding provider is OpenAI.+ """+ from config.settings import get_openrag_config++ config = get_openrag_config()+ checked: set[str] = set()++ candidates: list[tuple[str | None, object, str | None, str | None]] = [+ (+ config.knowledge.embedding_provider,+ config.get_embedding_provider_config(),+ config.knowledge.embedding_model,+ None,+ ),+ (+ config.agent.llm_provider,+ config.get_llm_provider_config(),+ None,+ config.agent.llm_model,+ ),+ ]+ for name in ("openai", "anthropic", "watsonx", "ollama"):+ candidates.append((name, getattr(config.providers, name, None), None, None))++ for provider, provider_config, embedding_model, llm_model in candidates:+ if not provider or provider in checked or provider_config is None:+ continue+ checked.add(provider)+ api_key = getattr(provider_config, "api_key", None)+ endpoint = getattr(provider_config, "endpoint", None)+ if provider == "ollama":+ if not endpoint:+ continue+ elif (+ provider+ not in (+ config.knowledge.embedding_provider,+ config.agent.llm_provider,+ )+ and not api_key+ ):+ continue++ error = await _probe_provider_credential_error(+ provider=provider,+ api_key=api_key,+ endpoint=endpoint,+ project_id=getattr(provider_config, "project_id", None),+ embedding_model=embedding_model,+ llm_model=llm_model,+ )+ if error:+ return error+ return None+++async def resolve_ingest_error_message(exc: BaseException | str) -> str:+ """Prefer a credential message when Langflow fails auth or only reports a disconnect."""+ raw = (str(exc) if not isinstance(exc, str) else exc).strip() or "Ingestion failed"++ cleaned = sanitize_provider_error_content(raw)+ if is_provider_credential_error(raw) or is_provider_credential_error(cleaned):+ return cleaned++ if is_langflow_transport_error(raw):+ probe = await probe_provider_credential_error()+ if probe:+ logger.info(+ "Ingest transport failure attributed to provider credentials",+ transport_error=raw,+ credential_error=probe,+ )+ return probe++ # Prefer the sanitized form whenever JSON / boilerplate was stripped.+ if cleaned != raw and "{" not in cleaned:+ return cleaned+ return raw+++def resolve_chat_stream_error_message(text: str | BaseException | None) -> str:+ """Sanitize chat stream errors without probing providers.++ Do not infer API-key failures from opaque upstream text — that can hide the+ real failure. Callers should break Langflow session chaining after errors so+ the next turn is a fresh session that returns the detailed upstream message.+ """+ raw = (str(text) if not isinstance(text, str) else text) if text is not None else ""+ if not raw.strip():+ return "An error occurred while generating a response."+ return sanitize_provider_error_content(raw)++def _parse_json_error_message(error_text: str) -> str:"""Parse JSON error message and extract just the message field."""try:# Try to parse as JSONerror_data = json.loads(error_text)-+if isinstance(error_data, dict):# WatsonX format: {"errors": [{"code": "...", "message": "..."}], ...}if "errors" in error_data and isinstance(error_data["errors"], list):@@ -23,7 +342,7 @@ def _parse_json_error_message(error_text: str) -> str:code = errors[0].get("code", "")if code:return f"Error: {code}"-+# OpenAI format: {"error": {"message": "...", "type": "...", "code": "..."}}if "error" in error_data:error_obj = error_data["error"]@@ -31,17 +350,21 @@ def _parse_json_error_message(error_text: str) -> str:message = error_obj.get("message", "")if message:return message-+# Direct message fieldif "message" in error_data:return error_data["message"]-++ # IBM IAM format: {"errorCode": "...", "errorMessage": "...", "context": {...}}+ if "errorMessage" in error_data:+ return error_data["errorMessage"]+# Generic format: {"detail": "..."}if "detail" in error_data:return error_data["detail"]except (json.JSONDecodeError, ValueError, TypeError):pass-+# Return original text if not JSON or can't parsereturn error_text@@ -51,7 +374,7 @@ def _extract_error_details(response: httpx.Response) -> str:try:# Try to parse JSON error responseerror_data = response.json()-+# Common error response formatsif isinstance(error_data, dict):# WatsonX format: {"errors": [{"code": "...", "message": "..."}], ...}@@ -66,7 +389,7 @@ def _extract_error_details(response: httpx.Response) -> str:code = errors[0].get("code", "")if code:return f"Error: {code}"-+# OpenAI format: {"error": {"message": "...", "type": "...", "code": "..."}}if "error" in error_data:error_obj = error_data["error"]@@ -81,22 +404,26 @@ def _extract_error_details(response: httpx.Response) -> str:if code:details += f" (code: {code})"return details-- # Anthropic format: {"error": {"message": "...", "type": "..."}}++ # Anthropic / generic: {"message": "..."}if "message" in error_data:return error_data["message"]-- # Generic format: {"message": "..."}++ # IBM IAM format: {"errorCode": "...", "errorMessage": "...", "context": {...}}+ if "errorMessage" in error_data:+ return error_data["errorMessage"]++ # Generic format: {"detail": "..."}if "detail" in error_data:return error_data["detail"]-+# If JSON parsing worked but no structured error found, try parsing textresponse_text = response.text[:500]parsed = _parse_json_error_message(response_text)if parsed != response_text:return parsedreturn response_text-+except (json.JSONDecodeError, ValueError):# If JSON parsing fails, try parsing the text as JSON stringresponse_text = response.text[:500] if response.text else f"HTTP {response.status_code}"@@ -134,7 +461,9 @@ async def validate_provider_setup(provider_lower = provider.lower()try:- logger.info(f"Starting validation for provider: {provider_lower} (test_completion={test_completion})")+ logger.info(+ f"Starting validation for provider: {provider_lower} (test_completion={test_completion})"+ )if test_completion:# Full validation with completion/embedding tests (consumes credits)@@ -236,7 +565,7 @@ async def test_embedding(# OpenAI validation functionsasync def _test_openai_lightweight_health(api_key: str) -> None:"""Test OpenAI API key validity with lightweight check.-+Only checks if the API key is valid without consuming credits.Uses the /v1/models endpoint which doesn't consume credits."""@@ -256,14 +585,16 @@ async def _test_openai_lightweight_health(api_key: str) -> None:if response.status_code != 200:error_details = _extract_error_details(response)- logger.error(f"OpenAI lightweight health check failed: {response.status_code} - {error_details}")+ logger.error(+ f"OpenAI lightweight health check failed: {response.status_code} - {error_details}"+ )raise Exception(f"OpenAI API key validation failed: {error_details}")logger.info("OpenAI lightweight health check passed")except httpx.TimeoutException:logger.error("OpenAI lightweight health check timed out")- raise Exception("OpenAI API request timed out")+ raise Exception("OpenAI API request timed out") from Noneexcept Exception as e:logger.error(f"OpenAI lightweight health check failed: {str(e)}")raise@@ -280,9 +611,7 @@ async def _test_openai_completion_with_tools(api_key: str, llm_model: str) -> No# Simple tool calling testbase_payload = {"model": llm_model,- "messages": [- {"role": "user", "content": "What tools do you have available?"}- ],+ "messages": [{"role": "user", "content": "What tools do you have available?"}],"tools": [{"type": "function",@@ -292,14 +621,11 @@ async def _test_openai_completion_with_tools(api_key: str, llm_model: str) -> No"parameters": {"type": "object","properties": {- "location": {- "type": "string",- "description": "The city and state"- }+ "location": {"type": "string", "description": "The city and state"}},- "required": ["location"]- }- }+ "required": ["location"],+ },+ },}],}@@ -316,7 +642,9 @@ async def _test_openai_completion_with_tools(api_key: str, llm_model: str) -> No# If max_tokens doesn't work, try with max_completion_tokensif response.status_code != 200:- logger.warning("[API] max_tokens parameter failed, trying max_completion_tokens instead")+ logger.warning(+ "[API] max_tokens parameter failed, trying max_completion_tokens instead"+ )payload = {**base_payload, "max_completion_tokens": 50}response = await client.post("https://api.openai.com/v1/chat/completions",@@ -327,14 +655,16 @@ async def _test_openai_completion_with_tools(api_key: str, llm_model: str) -> Noif response.status_code != 200:error_details = _extract_error_details(response)- logger.error(f"OpenAI completion test failed: {response.status_code} - {error_details}")+ logger.error(+ f"OpenAI completion test failed: {response.status_code} - {error_details}"+ )raise Exception(f"OpenAI API error: {error_details}")logger.info("OpenAI completion with tool calling test passed")except httpx.TimeoutException:logger.error("OpenAI completion test timed out")- raise Exception("Request timed out")+ raise Exception("Request timed out") from Noneexcept Exception as e:logger.error(f"OpenAI completion test failed: {str(e)}")raise@@ -363,7 +693,9 @@ async def _test_openai_embedding(api_key: str, embedding_model: str) -> None:if response.status_code != 200:error_details = _extract_error_details(response)- logger.error(f"OpenAI embedding test failed: {response.status_code} - {error_details}")+ logger.error(+ f"OpenAI embedding test failed: {response.status_code} - {error_details}"+ )raise Exception(f"OpenAI API error: {error_details}")data = response.json()@@ -374,18 +706,16 @@ async def _test_openai_embedding(api_key: str, embedding_model: str) -> None:except httpx.TimeoutException:logger.error("OpenAI embedding test timed out")- raise Exception("Request timed out")+ raise Exception("Request timed out") from Noneexcept Exception as e:logger.error(f"OpenAI embedding test failed: {str(e)}")raise# IBM Watson validation functions-async def _test_watsonx_lightweight_health(- api_key: str, endpoint: str, project_id: str-) -> None:+async def _test_watsonx_lightweight_health(api_key: str, endpoint: str, project_id: str) -> None:"""Test WatsonX API key validity with lightweight check.-+Only checks if the API key is valid by getting a bearer token.Does not consume credits by avoiding model inference requests."""@@ -404,7 +734,9 @@ async def _test_watsonx_lightweight_health(if token_response.status_code != 200:error_details = _extract_error_details(token_response)- logger.error(f"IBM IAM token request failed: {token_response.status_code} - {error_details}")+ logger.error(+ f"IBM IAM token request failed: {token_response.status_code} - {error_details}"+ )raise Exception(f"Failed to authenticate with IBM Watson: {error_details}")bearer_token = token_response.json().get("access_token")@@ -415,7 +747,7 @@ async def _test_watsonx_lightweight_health(except httpx.TimeoutException:logger.error("WatsonX lightweight health check timed out")- raise Exception("WatsonX API request timed out")+ raise Exception("WatsonX API request timed out") from Noneexcept Exception as e:logger.error(f"WatsonX lightweight health check failed: {str(e)}")raise@@ -440,7 +772,9 @@ async def _test_watsonx_completion_with_tools(if token_response.status_code != 200:error_details = _extract_error_details(token_response)- logger.error(f"IBM IAM token request failed: {token_response.status_code} - {error_details}")+ logger.error(+ f"IBM IAM token request failed: {token_response.status_code} - {error_details}"+ )raise Exception(f"Failed to authenticate with IBM Watson: {error_details}")bearer_token = token_response.json().get("access_token")@@ -458,9 +792,7 @@ async def _test_watsonx_completion_with_tools(payload = {"model_id": llm_model,"project_id": project_id,- "messages": [- {"role": "user", "content": "What tools do you have available?"}- ],+ "messages": [{"role": "user", "content": "What tools do you have available?"}],"tools": [{"type": "function",@@ -470,14 +802,11 @@ async def _test_watsonx_completion_with_tools("parameters": {"type": "object","properties": {- "location": {- "type": "string",- "description": "The city and state"- }+ "location": {"type": "string", "description": "The city and state"}},- "required": ["location"]- }- }+ "required": ["location"],+ },+ },}],"max_tokens": 50,@@ -494,7 +823,9 @@ async def _test_watsonx_completion_with_tools(if response.status_code != 200:error_details = _extract_error_details(response)- logger.error(f"IBM Watson completion test failed: {response.status_code} - {error_details}")+ logger.error(+ f"IBM Watson completion test failed: {response.status_code} - {error_details}"+ )# If error_details is still JSON, parse it to extract just the messageparsed_details = _parse_json_error_message(error_details)raise Exception(f"IBM Watson API error: {parsed_details}")@@ -503,7 +834,7 @@ async def _test_watsonx_completion_with_tools(except httpx.TimeoutException:logger.error("IBM Watson completion test timed out")- raise Exception("Request timed out")+ raise Exception("Request timed out") from Noneexcept Exception as e:logger.error(f"IBM Watson completion test failed: {str(e)}")# If the error message contains JSON, parse it to extract just the message@@ -512,7 +843,7 @@ async def _test_watsonx_completion_with_tools(json_part = error_str.split("IBM Watson API error: ", 1)[1]parsed_message = _parse_json_error_message(json_part)if parsed_message != json_part:- raise Exception(f"IBM Watson API error: {parsed_message}")+ raise Exception(f"IBM Watson API error: {parsed_message}") from eraise@@ -535,7 +866,9 @@ async def _test_watsonx_embedding(if token_response.status_code != 200:error_details = _extract_error_details(token_response)- logger.error(f"IBM IAM token request failed: {token_response.status_code} - {error_details}")+ logger.error(+ f"IBM IAM token request failed: {token_response.status_code} - {error_details}"+ )raise Exception(f"Failed to authenticate with IBM Watson: {error_details}")bearer_token = token_response.json().get("access_token")@@ -567,7 +900,9 @@ async def _test_watsonx_embedding(if response.status_code != 200:error_details = _extract_error_details(response)- logger.error(f"IBM Watson embedding test failed: {response.status_code} - {error_details}")+ logger.error(+ f"IBM Watson embedding test failed: {response.status_code} - {error_details}"+ )# If error_details is still JSON, parse it to extract just the messageparsed_details = _parse_json_error_message(error_details)raise Exception(f"IBM Watson API error: {parsed_details}")@@ -580,7 +915,7 @@ async def _test_watsonx_embedding(except httpx.TimeoutException:logger.error("IBM Watson embedding test timed out")- raise Exception("Request timed out")+ raise Exception("Request timed out") from Noneexcept Exception as e:logger.error(f"IBM Watson embedding test failed: {str(e)}")# If the error message contains JSON, parse it to extract just the message@@ -589,14 +924,14 @@ async def _test_watsonx_embedding(json_part = error_str.split("IBM Watson API error: ", 1)[1]parsed_message = _parse_json_error_message(json_part)if parsed_message != json_part:- raise Exception(f"IBM Watson API error: {parsed_message}")+ raise Exception(f"IBM Watson API error: {parsed_message}") from eraise# Ollama validation functionsasync def _test_ollama_lightweight_health(endpoint: str) -> None:"""Test Ollama availability with lightweight status check.-+Only checks if the endpoint returns a 200 status without fetching data."""try:@@ -610,14 +945,16 @@ async def _test_ollama_lightweight_health(endpoint: str) -> None:if response.status_code != 200:error_details = _extract_error_details(response)- logger.error(f"Ollama lightweight health check failed: {response.status_code} - {error_details}")+ logger.error(+ f"Ollama lightweight health check failed: {response.status_code} - {error_details}"+ )raise Exception(f"Ollama endpoint not responding: {error_details}")logger.info("Ollama lightweight health check passed")except httpx.TimeoutException:logger.error("Ollama lightweight health check timed out")- raise Exception("Ollama endpoint timed out")+ raise Exception("Ollama endpoint timed out") from Noneexcept Exception as e:logger.error(f"Ollama lightweight health check failed: {str(e)}")raise@@ -631,9 +968,7 @@ async def _test_ollama_completion_with_tools(llm_model: str, endpoint: str) -> Npayload = {"model": llm_model,- "messages": [- {"role": "user", "content": "What tools do you have available?"}- ],+ "messages": [{"role": "user", "content": "What tools do you have available?"}],"tools": [{"type": "function",@@ -643,14 +978,11 @@ async def _test_ollama_completion_with_tools(llm_model: str, endpoint: str) -> N"parameters": {"type": "object","properties": {- "location": {- "type": "string",- "description": "The city and state"- }+ "location": {"type": "string", "description": "The city and state"}},- "required": ["location"]- }- }+ "required": ["location"],+ },+ },}],"stream": False,@@ -665,14 +997,16 @@ async def _test_ollama_completion_with_tools(llm_model: str, endpoint: str) -> Nif response.status_code != 200:error_details = _extract_error_details(response)- logger.error(f"Ollama completion test failed: {response.status_code} - {error_details}")+ logger.error(+ f"Ollama completion test failed: {response.status_code} - {error_details}"+ )raise Exception(f"Ollama API error: {error_details}")logger.info("Ollama completion with tool calling test passed")except httpx.TimeoutException:logger.error("Ollama completion test timed out")- raise httpx.TimeoutException("Ollama is busy or model inference timed out")+ raise httpx.TimeoutException("Ollama is busy or model inference timed out") from Noneexcept Exception as e:logger.error(f"Ollama completion test failed: {str(e)}")raise@@ -698,7 +1032,9 @@ async def _test_ollama_embedding(embedding_model: str, endpoint: str) -> None:if response.status_code != 200:error_details = _extract_error_details(response)- logger.error(f"Ollama embedding test failed: {response.status_code} - {error_details}")+ logger.error(+ f"Ollama embedding test failed: {response.status_code} - {error_details}"+ )raise Exception(f"Ollama API error: {error_details}")data = response.json()@@ -709,7 +1045,7 @@ async def _test_ollama_embedding(embedding_model: str, endpoint: str) -> None:except httpx.TimeoutException:logger.error("Ollama embedding test timed out")- raise httpx.TimeoutException("Ollama is busy or embedding generation timed out")+ raise httpx.TimeoutException("Ollama is busy or embedding generation timed out") from Noneexcept Exception as e:logger.error(f"Ollama embedding test failed: {str(e)}")raise@@ -718,7 +1054,7 @@ async def _test_ollama_embedding(embedding_model: str, endpoint: str) -> None:# Anthropic validation functionsasync def _test_anthropic_lightweight_health(api_key: str) -> None:"""Test Anthropic API key validity with lightweight check.-+Only checks if the API key is valid without consuming credits.Uses the /v1/models endpoint which doesn't consume credits."""@@ -738,14 +1074,16 @@ async def _test_anthropic_lightweight_health(api_key: str) -> None:if response.status_code != 200:error_details = _extract_error_details(response)- logger.error(f"Anthropic lightweight health check failed: {response.status_code} - {error_details}")+ logger.error(+ f"Anthropic lightweight health check failed: {response.status_code} - {error_details}"+ )raise Exception(f"Anthropic API key validation failed: {error_details}")logger.info("Anthropic lightweight health check passed")except httpx.TimeoutException:logger.error("Anthropic lightweight health check timed out")- raise Exception("Anthropic API request timed out")+ raise Exception("Anthropic API request timed out") from Noneexcept Exception as e:logger.error(f"Anthropic lightweight health check failed: {str(e)}")raise@@ -764,9 +1102,7 @@ async def _test_anthropic_completion_with_tools(api_key: str, llm_model: str) ->payload = {"model": llm_model,"max_tokens": 50,- "messages": [- {"role": "user", "content": "What tools do you have available?"}- ],+ "messages": [{"role": "user", "content": "What tools do you have available?"}],"tools": [{"name": "get_weather",@@ -774,13 +1110,10 @@ async def _test_anthropic_completion_with_tools(api_key: str, llm_model: str) ->"input_schema": {"type": "object","properties": {- "location": {- "type": "string",- "description": "The city and state"- }+ "location": {"type": "string", "description": "The city and state"}},- "required": ["location"]- }+ "required": ["location"],+ },}],}@@ -795,14 +1128,16 @@ async def _test_anthropic_completion_with_tools(api_key: str, llm_model: str) ->if response.status_code != 200:error_details = _extract_error_details(response)- logger.error(f"Anthropic completion test failed: {response.status_code} - {error_details}")+ logger.error(+ f"Anthropic completion test failed: {response.status_code} - {error_details}"+ )raise Exception(f"Anthropic API error: {error_details}")logger.info("Anthropic completion with tool calling test passed")except httpx.TimeoutException:logger.error("Anthropic completion test timed out")- raise Exception("Request timed out")+ raise Exception("Request timed out") from Noneexcept Exception as e:logger.error(f"Anthropic completion test failed: {str(e)}")raisediff --git a/src/services/chat_service.py b/src/services/chat_service.pyindex 1b89f7a73..bf9749dd9 100644--- a/src/services/chat_service.py+++ b/src/services/chat_service.py@@ -59,6 +59,7 @@ class ChatService:user_id: str = None,jwt_token: str = None,previous_response_id: str = None,+ conversation_id: str = None,stream: bool = False,filter_id: str = None,owner: str = None,@@ -215,6 +216,7 @@ class ChatService:conversation_user_id,extra_headers=extra_headers,previous_response_id=previous_response_id,+ conversation_id=conversation_id,filter_id=filter_id,)else:@@ -227,6 +229,7 @@ class ChatService:conversation_user_id,extra_headers=extra_headers,previous_response_id=previous_response_id,+ conversation_id=conversation_id,filter_id=filter_id,)response_data = {"response": response_text}@@ -701,20 +704,45 @@ class ChatService:"total_conversations": len(conversations),}+ @staticmethod+ def _messages_from_active_conversation(conversation_state: dict) -> list[dict]:+ """Serialize in-memory OpenRAG thread messages for the history API."""+ messages = []+ for msg in conversation_state.get("messages", []):+ if msg.get("role") not in ["user", "assistant"]:+ continue+ message_data = {+ "role": msg["role"],+ "content": msg["content"],+ "timestamp": msg.get("timestamp").isoformat()+ if hasattr(msg.get("timestamp"), "isoformat")+ else msg.get("timestamp"),+ "source": "openrag_memory",+ }+ if msg.get("error"):+ message_data["error"] = True+ if msg.get("chunks"):+ message_data["chunks"] = msg["chunks"]+ if msg.get("response_data"):+ message_data["response_data"] = msg["response_data"]+ messages.append(message_data)+ return messages+async def get_langflow_history(self, user_id: str):"""Get langflow conversation history for a user - now fetches from both OpenRAG memory and Langflow database"""- from agent import get_user_conversations+ from agent import active_conversations, get_user_conversationsfrom services.langflow_history_service import langflow_history_serviceif not user_id:return {"error": "User ID is required", "conversations": []}all_conversations = []+ local_metadata: dict = {}+ in_memory = active_conversations.get(user_id, {})try:# 1. Get local conversation metadata (no actual messages stored here)conversations_dict = await get_user_conversations(user_id)- local_metadata = {}for response_id, conversation_metadata in conversations_dict.items():# Store metadata for later use with Langflow data@@ -756,56 +784,102 @@ class ChatService:messages.append(message_data)+ # Prefer OpenRAG memory when it has more turns (error retries use a+ # fresh Langflow session under the same OpenRAG thread id).+ memory_messages = self._messages_from_active_conversation(+ in_memory.get(session_id, {})+ )+ if len(memory_messages) > len(messages):+ messages = memory_messages+ source = "openrag_memory_preferred"+ else:+ source = "langflow_enhanced"+if messages:- # Use local metadata if available, otherwise generate from Langflow datametadata = local_metadata.get(session_id, {})-- if not metadata.get("title"):- first_user_msg = next(- (msg for msg in messages if msg["role"] == "user"), None- )- title = (- first_user_msg["content"][:50] + "..."- if first_user_msg and len(first_user_msg["content"]) > 50- else first_user_msg["content"]- if first_user_msg- else "Langflow chat"- )- else:- title = metadata["title"]+ first_user_msg = next(+ (msg for msg in messages if msg["role"] == "user"), None+ )+ title = metadata.get("title") or (+ first_user_msg["content"][:50] + "..."+ if first_user_msg and len(first_user_msg["content"]) > 50+ else first_user_msg["content"]+ if first_user_msg+ else "Langflow chat"+ )all_conversations.append({"response_id": session_id,"title": title,"endpoint": "langflow",- "messages": messages, # Function calls preserved from Langflow+ "messages": messages,"created_at": metadata.get("created_at")or conversation.get("created_at"),"last_activity": metadata.get("last_activity")or conversation.get("last_activity"),"filter_id": metadata.get("filter_id"),"total_messages": len(messages),- "source": "langflow_enhanced",+ "source": source,"langflow_session_id": session_id,"langflow_flow_id": conversation.get("flow_id"),})- if langflow_history.get("conversations"):- logger.debug(- f"Added {len(langflow_history['conversations'])} historical conversations from Langflow"- )- elif langflow_history.get("error"):- logger.debug(- f"Could not fetch Langflow history for user {user_id}: {langflow_history['error']}"- )- else:- logger.debug(f"No Langflow conversations found for user {user_id}")-except Exception as e:logger.error(f"Failed to fetch Langflow history: {e}")- # Continue with just in-memory conversations++ # Local-only threads + upgrade listed ones from memory/metadata snapshots+ # (Langflow history is incomplete after fresh-session error retries).+ listed_ids = {c["response_id"] for c in all_conversations}+ for response_id, metadata in local_metadata.items():+ conversation_state = in_memory.get(response_id, {})+ messages = self._messages_from_active_conversation(conversation_state)+ if not messages and isinstance(metadata.get("messages"), list):+ messages = [+ m+ for m in metadata["messages"]+ if isinstance(m, dict) and m.get("role") in ["user", "assistant"]+ ]++ if response_id in listed_ids:+ existing = next(c for c in all_conversations if c["response_id"] == response_id)+ if len(messages) > len(existing.get("messages") or []):+ existing["messages"] = messages+ existing["total_messages"] = len(messages)+ existing["source"] = "openrag_memory_preferred"+ continue++ if not messages and not metadata.get("title"):+ continue++ first_user_msg = next((msg for msg in messages if msg["role"] == "user"), None)+ title = metadata.get("title") or (+ first_user_msg["content"][:50] + "..."+ if first_user_msg and len(first_user_msg["content"]) > 50+ else first_user_msg["content"]+ if first_user_msg+ else "New Chat"+ )+ created_at = metadata.get("created_at") or conversation_state.get("created_at")+ last_activity = metadata.get("last_activity") or conversation_state.get("last_activity")+ all_conversations.append(+ {+ "response_id": response_id,+ "title": title,+ "endpoint": "langflow",+ "messages": messages,+ "created_at": created_at.isoformat()+ if hasattr(created_at, "isoformat")+ else created_at,+ "last_activity": last_activity.isoformat()+ if hasattr(last_activity, "isoformat")+ else last_activity,+ "filter_id": metadata.get("filter_id") or conversation_state.get("filter_id"),+ "total_messages": len(messages) or metadata.get("total_messages", 0),+ "source": "local_error_fallback",+ }+ )# Sort by last activity (most recent first)all_conversations.sort(key=lambda c: c.get("last_activity", ""), reverse=True)diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.pyindex ff560aca7..48e36330e 100644--- a/src/services/langflow_file_service.py+++ b/src/services/langflow_file_service.py@@ -4,7 +4,7 @@ import jsonimport timeimport uuidfrom pathlib import Path-from typing import Any+from typing import Any, NoReturnimport httpx@@ -307,6 +307,15 @@ class LangflowFileService:error=str(e),)+ async def _raise_resolved_ingest_error(self, exc: BaseException) -> NoReturn:+ """Re-raise with a credential message when Langflow disconnects on bad API keys."""+ from api.provider_validation import resolve_ingest_error_message++ resolved = await resolve_ingest_error_message(exc)+ if resolved != (str(exc) or "").strip():+ raise Exception(resolved) from exc+ raise exc+async def upload_user_file(self, file_tuple, jwt_token: str | None = None) -> dict[str, Any]:"""Upload a file using Langflow Files API v2: POST /api/v2/files.Returns JSON with keys: id, name, path, size, provider.@@ -589,12 +598,12 @@ class LangflowFileService:raisereturn resp_json- except Exception:+ except Exception as e:await self._cleanup_failed_callback_ingest(ingest_token=ingest_token,ingest_run_id=ingest_run_id,)- raise+ await self._raise_resolved_ingest_error(e)async def run_url_ingestion_flow(self,@@ -724,12 +733,12 @@ class LangflowFileService:)return resp.json()- except Exception:+ except Exception as e:await self._cleanup_failed_callback_ingest(ingest_token=ingest_token,ingest_run_id=ingest_run_id,)- raise+ await self._raise_resolved_ingest_error(e)async def _ensure_url_ingest_flow_id(self) -> str:"""Ensure URL ingest flow ID is valid; import flow if missing.diff --git a/src/services/langflow_history_service.py b/src/services/langflow_history_service.pyindex dc4111486..35a2405cc 100644--- a/src/services/langflow_history_service.py+++ b/src/services/langflow_history_service.py@@ -3,32 +3,35 @@ Langflow Message History ServiceSimplified service that retrieves message history from Langflow using shared client infrastructure"""-from typing import List, Dict, Optional, Any+from typing import Any+from api.provider_validation import (+ looks_like_provider_error_content,+ sanitize_provider_error_content,+)from config.settings import clientsfrom utils.logging_config import get_loggerlogger = get_logger(__name__)+class LangflowHistoryService:"""Simplified service to retrieve message history from Langflow"""-+def __init__(self):pass-- async def get_user_sessions(self, user_id: str, flow_id: Optional[str] = None) -> List[str]:++ async def get_user_sessions(self, user_id: str, flow_id: str | None = None) -> list[str]:"""Get all session IDs for a user's conversations"""try:params = {}if flow_id:params["flow_id"] = flow_id-+response = await clients.langflow_request(- "GET",- "/api/v1/monitor/messages/sessions",- params=params+ "GET", "/api/v1/monitor/messages/sessions", params=params)-+if response.status_code == 200:session_ids = response.json()logger.debug(f"Found {len(session_ids)} total sessions from Langflow")@@ -36,65 +39,73 @@ class LangflowHistoryService:else:logger.error(f"Failed to get sessions: {response.status_code} - {response.text}")return []-+except Exception as e:logger.error(f"Error getting user sessions: {e}")return []-- async def get_session_messages(self, user_id: str, session_id: str) -> List[Dict[str, Any]]:++ async def get_session_messages(self, user_id: str, session_id: str) -> list[dict[str, Any]]:"""Get all messages for a specific session"""try:response = await clients.langflow_request("GET","/api/v1/monitor/messages",- params={- "session_id": session_id,- "order_by": "timestamp"- }+ params={"session_id": session_id, "order_by": "timestamp"},)-+if response.status_code == 200:messages = response.json()# Convert to OpenRAG formatreturn self._convert_langflow_messages(messages)else:- logger.error(f"Failed to get messages for session {session_id}: {response.status_code}")+ logger.error(+ f"Failed to get messages for session {session_id}: {response.status_code}"+ )return []-+except Exception as e:logger.error(f"Error getting session messages: {e}")return []-- def _convert_langflow_messages(self, langflow_messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:++ def _convert_langflow_messages(+ self, langflow_messages: list[dict[str, Any]]+ ) -> list[dict[str, Any]]:"""Convert Langflow messages to OpenRAG format"""- converted_messages = []-+ converted_messages: list[dict[str, Any]] = []+for msg in langflow_messages:try:- content = msg.get("text", "")-- # Detect error messages - check explicit error flag or common error patterns- is_error = msg.get("error", False) or (- msg.get("sender") != "User" and content.startswith("Error:")- )-+ content = msg.get("text", "") or ""+ sender = msg.get("sender")+ is_user = sender == "User"++ # Detect error messages - explicit flag, Error: prefix, or provider failures+ # (including raw JSON payloads that leak from upstream auth/rate-limit errors).+ is_error = False+ if not is_user:+ is_error = bool(msg.get("error", False)) or looks_like_provider_error_content(+ content+ )+ if is_error:+ content = sanitize_provider_error_content(content)+# Map Langflow message format to OpenRAG formatconverted_msg = {- "role": "user" if msg.get("sender") == "User" else "assistant",+ "role": "user" if is_user else "assistant","content": content,"timestamp": msg.get("timestamp"),"langflow_message_id": msg.get("id"),"langflow_session_id": msg.get("session_id"),"langflow_flow_id": msg.get("flow_id"),- "sender": msg.get("sender"),+ "sender": sender,"sender_name": msg.get("sender_name"),"source": "langflow","files": msg.get("files", []),"properties": msg.get("properties", {}),"error": is_error,- "edit": msg.get("edit", False)+ "edit": msg.get("edit", False),}-+# Extract function calls from content_blocks if present# Convert to match streaming format: chunk.item.type === "tool_call"content_blocks = msg.get("content_blocks", [])@@ -102,43 +113,59 @@ class LangflowHistoryService:chunks = []for block in content_blocks:if block.get("title") == "Agent Steps" and block.get("contents"):- for content in block["contents"]:- if content.get("type") == "tool_use":+ for content_item in block["contents"]:+ if content_item.get("type") == "tool_use":# Convert Langflow tool_use format to match streaming chunks format# Frontend expects: chunk.item.type === "tool_call" with tool_name, inputs, resultschunk = {"type": "response.output_item.added","item": {"type": "tool_call",- "tool_name": content.get("name", ""),- "inputs": content.get("tool_input", {}),- "results": content.get("output", {}),- "id": content.get("id") or content.get("run_id", ""),- "status": "completed" if not content.get("error") else "error"- }+ "tool_name": content_item.get("name", ""),+ "inputs": content_item.get("tool_input", {}),+ "results": content_item.get("output", {}),+ "id": content_item.get("id")+ or content_item.get("run_id", ""),+ "status": "completed"+ if not content_item.get("error")+ else "error",+ },}chunks.append(chunk)-+if chunks:converted_msg["chunks"] = chunks-++ # Langflow often records the same provider failure several times+ # in one session (e.g. embedding + LLM auth retries). Keep one.+ prev = converted_messages[-1] if converted_messages else None+ if (+ is_error+ and prev+ and prev.get("error")+ and (prev.get("content") or "").strip() == content.strip()+ ):+ continue+converted_messages.append(converted_msg)-+except Exception as e:logger.error(f"Error converting message: {e}")continue-+return converted_messages-- async def get_user_conversation_history(self, user_id: str, flow_id: Optional[str] = None) -> Dict[str, Any]:++ async def get_user_conversation_history(+ self, user_id: str, flow_id: str | None = None+ ) -> dict[str, Any]:"""Get all conversation history for a user, organized by session-+Simplified version - gets all sessions and lets the frontend filter by user_id"""try:# Get all sessions (no complex filtering needed)session_ids = await self.get_user_sessions(user_id, flow_id)-+conversations = []for session_id in session_ids:messages = await self.get_session_messages(user_id, session_id)@@ -146,7 +173,7 @@ class LangflowHistoryService:# Create conversation metadatafirst_message = messages[0] if messages else Nonelast_message = messages[-1] if messages else None-+conversation = {"session_id": session_id,"langflow_session_id": session_id, # For compatibility@@ -156,26 +183,24 @@ class LangflowHistoryService:"created_at": first_message.get("timestamp") if first_message else None,"last_activity": last_message.get("timestamp") if last_message else None,"flow_id": first_message.get("langflow_flow_id") if first_message else None,- "source": "langflow"+ "source": "langflow",}conversations.append(conversation)-- # Sort by last activity (most recent first)- conversations.sort(key=lambda c: c.get("last_activity", ""), reverse=True)-++ # Sort by last activity (most recent first). Normalize None so mixed+ # None/str timestamps do not raise TypeError during comparison.+ conversations.sort(key=lambda c: c.get("last_activity") or "", reverse=True)+return {"conversations": conversations,"total_conversations": len(conversations),- "user_id": user_id+ "user_id": user_id,}-+except Exception as e:logger.error(f"Error getting user conversation history: {e}")- return {- "error": str(e),- "conversations": []- }+ return {"error": str(e), "conversations": []}# Global instance-langflow_history_service = LangflowHistoryService()\ No newline at end of file+langflow_history_service = LangflowHistoryService()diff --git a/src/services/task_service.py b/src/services/task_service.pyindex 78e7e7002..7739d868e 100644--- a/src/services/task_service.py+++ b/src/services/task_service.py@@ -91,6 +91,28 @@ def _is_task_cancellation_error(error: str) -> bool:return any(marker in lowered for marker in _TASK_CANCELLATION_ERROR_MARKERS)+def _provider_credential_failure_metadata(error: str) -> dict | None:+ """Classify invalid/revoked provider API key failures for ingestion toasts/UI."""+ if not error:+ return None++ from api.provider_validation import (+ is_provider_credential_error,+ sanitize_provider_error_content,+ )++ cleaned = sanitize_provider_error_content(error)+ if not (is_provider_credential_error(error) or is_provider_credential_error(cleaned)):+ return None++ return {+ "component": "openrag",+ "failure_phase": "embedding",+ "user_facing_message": cleaned,+ "actionable_by": "USER_ACTIONABLE",+ }++def _is_transient_connectivity_error(error: str) -> bool:lowered = error.lower()return any(marker in lowered for marker in _TRANSIENT_CONNECTIVITY_ERROR_MARKERS)@@ -1088,6 +1110,14 @@ class TaskService:"actionable_by": "USER_ACTIONABLE",}+ # Prefer credential failures over transport messages. Langflow may report+ # "server disconnected" when the real cause is a revoked embedding API key;+ # resolve_ingest_error_message() rewrites file_task.error in that case, and+ # this check also covers errors that mention both disconnect + API key text.+ credential_meta = _provider_credential_failure_metadata(error)+ if credential_meta:+ return credential_meta+if _is_transient_connectivity_error(error) or _is_langflow_transport_failure(error):return {"component": "langflow",diff --git a/tests/unit/test_langflow_stream_provider_errors.py b/tests/unit/test_langflow_stream_provider_errors.pynew file mode 100644index 000000000..15fa3c504--- /dev/null+++ b/tests/unit/test_langflow_stream_provider_errors.py@@ -0,0 +1,453 @@+"""Provider errors must reach the client as a final stream error chunk.++These tests mock the upstream stream to raise exceptions shaped like real+provider failures (watsonx rate limit, invalid key, permission). That proves+the yield path without hardcoding a raise in production code.+"""++from __future__ import annotations++import json+from collections.abc import AsyncIterator++import pytest++import agent as agent_module+++def test_format_provider_error_message_extracts_embedded_json():+ raw = (+ 'Provider request failed: {"error": {"message": '+ '"Incorrect API key provided", "type": "invalid_request_error"}}'+ )+ assert (+ agent_module._format_provider_error_message(Exception(raw)) == "Incorrect API key provided"+ )+++def test_format_provider_error_message_extracts_ibm_iam_error_message():+ raw = (+ 'Failed to authenticate with IBM Watson: {"errorCode":"BXNIM0415E",'+ '"errorMessage":"Provided API key could not be found.",'+ '"context":{"requestId":"abc","url":"https://iam.cloud.ibm.com"}}'+ )+ assert agent_module._format_provider_error_message(Exception(raw)) == (+ "Provided API key could not be found."+ )+++def test_format_provider_error_message_keeps_plain_text():+ msg = "Rate limit exceeded for watsonx.ai. Please try again later."+ assert agent_module._format_provider_error_message(Exception(msg)) == msg+++def test_provider_error_display_text_keeps_partial_answer():+ assert agent_module._provider_error_display_text(+ "Rate limit exceeded", "Partial answer so far"+ ) == ("Partial answer so far\n\nRate limit exceeded")+++def test_provider_error_display_text_drops_error_like_partial():+ partial = (+ "Failed to initialize IBM WatsonX embedding model. Error: "+ '{"errorMessage":"Provided API key could not be found."}'+ )+ assert (+ agent_module._provider_error_display_text("Provided API key could not be found.", partial)+ == "Provided API key could not be found."+ )+++def test_resolve_conversation_store_id_prefers_warm_thread():+ cold_id = agent_module._resolve_conversation_store_id(+ None,+ "resp_cold",+ previous_loaded_from_memory=False,+ mint_if_missing=True,+ )+ # Cold previous ids must not be reused (would overwrite metadata); mint a new id.+ assert cold_id != "resp_cold"+ assert not cold_id.startswith("error_")++ assert (+ agent_module._resolve_conversation_store_id(+ None,+ "resp_warm",+ previous_loaded_from_memory=True,+ )+ == "resp_warm"+ )+ assert (+ agent_module._resolve_conversation_store_id(+ "resp_stream",+ "resp_warm",+ previous_loaded_from_memory=True,+ )+ == "resp_warm"+ )+++def store_in_memory(monkeypatch):+ stored: list[tuple[str, str]] = []+ claimed: list[tuple[str, str]] = []++ async def _store(user_id, response_id, conversation_state):+ stored.append((user_id, response_id))+ if user_id not in agent_module.active_conversations:+ agent_module.active_conversations[user_id] = {}+ agent_module.active_conversations[user_id][response_id] = conversation_state++ class _Ownership:+ async def claim_session(self, user_id, session_id):+ claimed.append((user_id, session_id))++ monkeypatch.setattr(agent_module, "store_conversation_thread", _store)+ monkeypatch.setattr(+ "services.session_ownership_service.session_ownership_service",+ _Ownership(),+ )+ return {"stored": stored, "claimed": claimed}+++async def _collect_error_chunks(stream) -> list[dict]:+ chunks: list[dict] = []+ async for raw in stream:+ chunks.append(json.loads(raw.decode("utf-8")))+ return chunks+++ "provider_message",+ [+ "Rate limit exceeded for watsonx.ai. Please try again later.",+ "Invalid API key for Anthropic. Check your credentials.",+ "Permission denied: model access not authorized for this project.",+ "Quota exceeded: you have reached your monthly token limit.",+ ],+)+async def test_langflow_stream_yields_provider_error_chunk(+ monkeypatch, store_in_memory, provider_message: str+):+ async def raise_provider_error(*_args, **_kwargs) -> AsyncIterator[bytes]:+ raise Exception(provider_message)+ yield b"" # pragma: no cover — keep this an async generator++ monkeypatch.setattr(agent_module, "async_stream", raise_provider_error)++ user_id = "test-provider-error-user"+ agent_module.active_conversations.pop(user_id, None)++ chunks = await _collect_error_chunks(+ agent_module.async_langflow_chat_stream(+ langflow_client=object(),+ flow_id="flow-id",+ prompt="hello",+ user_id=user_id,+ )+ )++ assert len(chunks) == 1+ error_chunk = chunks[0]+ assert error_chunk["status"] == "failed"+ assert error_chunk["finish_reason"] == "error"+ assert error_chunk["error"]["message"] == provider_message++ # Failed first turns still get a real store id (UUID, not error_*).+ assert len(store_in_memory["stored"]) == 1+ stored_user, stored_id = store_in_memory["stored"][0]+ assert stored_user == user_id+ assert not stored_id.startswith("error_")+ assert error_chunk.get("id") == stored_id+ assert stored_id in agent_module.active_conversations.get(user_id, {})+++async def test_langflow_stream_persists_error_onto_existing_thread(monkeypatch, store_in_memory):+ provider_message = "Invalid API key for Anthropic. Check your credentials."++ async def raise_provider_error(*_args, **_kwargs) -> AsyncIterator[bytes]:+ raise Exception(provider_message)+ yield b"" # pragma: no cover++ monkeypatch.setattr(agent_module, "async_stream", raise_provider_error)++ user_id = "test-provider-error-existing"+ previous_id = "resp_existing_thread"+ agent_module.active_conversations[user_id] = {+ previous_id: {+ "messages": [{"role": "user", "content": "earlier"}],+ "created_at": None,+ "last_activity": None,+ }+ }++ chunks = await _collect_error_chunks(+ agent_module.async_langflow_chat_stream(+ langflow_client=object(),+ flow_id="flow-id",+ prompt="hello",+ user_id=user_id,+ previous_response_id=previous_id,+ )+ )++ assert chunks[0]["error"]["message"] == provider_message+ assert store_in_memory["stored"] == [(user_id, previous_id)]+ assert store_in_memory["claimed"] == [(user_id, previous_id)]+ assert not any(rid.startswith("error_") for _, rid in store_in_memory["stored"])++ stored = agent_module.active_conversations[user_id][previous_id]+ error_messages = [m for m in stored["messages"] if m.get("error")]+ assert len(error_messages) == 1+ assert error_messages[0]["content"] == provider_message+++async def test_langflow_follow_up_after_error_reuses_store_id(monkeypatch, store_in_memory):+ """Retrying in the same chat must not create a second sidebar conversation."""+ provider_message = "Rate limit exceeded for watsonx.ai."+ stream_calls: list[dict] = []++ async def raise_provider_error(*_args, **kwargs) -> AsyncIterator[bytes]:+ stream_calls.append(kwargs)+ raise Exception(provider_message)+ yield b"" # pragma: no cover++ monkeypatch.setattr(agent_module, "async_stream", raise_provider_error)++ user_id = "test-provider-error-follow-up"+ existing_id = "resp_failed_thread"+ agent_module.active_conversations[user_id] = {+ existing_id: {+ "messages": [+ {"role": "user", "content": "first"},+ {+ "role": "assistant",+ "content": "Provided API key could not be found.",+ "error": True,+ },+ ],+ "created_at": None,+ "last_activity": None,+ }+ }++ # Client keeps conversation_id after an error but omits previous_response_id+ # so Langflow starts fresh while OpenRAG appends to the same sidebar thread.+ chunks = await _collect_error_chunks(+ agent_module.async_langflow_chat_stream(+ langflow_client=object(),+ flow_id="flow-id",+ prompt="retry hello",+ user_id=user_id,+ conversation_id=existing_id,+ previous_response_id=None,+ )+ )++ assert chunks[0]["error"]["message"] == provider_message+ assert store_in_memory["stored"] == [(user_id, existing_id)]+ assert list(agent_module.active_conversations[user_id].keys()) == [existing_id]+ # Fresh Langflow session, same OpenRAG conversation id.+ assert stream_calls[0].get("previous_response_id") is None+ stored_messages = agent_module.active_conversations[user_id][existing_id]["messages"]+ assert [m["role"] for m in stored_messages if m["role"] != "system"][-2:] == [+ "user",+ "assistant",+ ]+++async def test_langflow_follow_up_ignores_broken_previous_response_id(monkeypatch, store_in_memory):+ """Even if the client still sends a failed Langflow session id, do not chain it."""+ provider_message = "Invalid API key for Anthropic. Check your credentials."+ stream_calls: list[dict] = []++ async def raise_provider_error(*_args, **kwargs) -> AsyncIterator[bytes]:+ stream_calls.append(kwargs)+ raise Exception(provider_message)+ yield b"" # pragma: no cover++ monkeypatch.setattr(agent_module, "async_stream", raise_provider_error)++ user_id = "test-provider-error-broken-session"+ existing_id = "resp_broken_langflow"+ agent_module.active_conversations[user_id] = {+ existing_id: {+ "messages": [+ {"role": "user", "content": "first"},+ {+ "role": "assistant",+ "content": provider_message,+ "error": True,+ },+ ],+ "created_at": None,+ "last_activity": None,+ }+ }++ chunks = await _collect_error_chunks(+ agent_module.async_langflow_chat_stream(+ langflow_client=object(),+ flow_id="flow-id",+ prompt="retry hello",+ user_id=user_id,+ conversation_id=existing_id,+ previous_response_id=existing_id,+ )+ )++ assert chunks[0]["error"]["message"] == provider_message+ assert stream_calls[0].get("previous_response_id") is None+ assert store_in_memory["stored"] == [(user_id, existing_id)]+++async def test_langflow_stream_skips_persist_for_cold_previous_response_id(+ monkeypatch, store_in_memory+):+ provider_message = "Permission denied: model access not authorized."++ async def raise_provider_error(*_args, **_kwargs) -> AsyncIterator[bytes]:+ raise Exception(provider_message)+ yield b"" # pragma: no cover++ monkeypatch.setattr(agent_module, "async_stream", raise_provider_error)++ user_id = "test-provider-error-cold"+ previous_id = "resp_not_in_memory"+ agent_module.active_conversations.pop(user_id, None)++ chunks = await _collect_error_chunks(+ agent_module.async_langflow_chat_stream(+ langflow_client=object(),+ flow_id="flow-id",+ prompt="hello",+ user_id=user_id,+ previous_response_id=previous_id,+ )+ )++ assert chunks[0]["error"]["message"] == provider_message+ # Cold previous_response_id must not overwrite conversation metadata —+ # persist under a freshly minted id instead.+ assert len(store_in_memory["stored"]) == 1+ stored_user, stored_id = store_in_memory["stored"][0]+ assert stored_user == user_id+ assert stored_id != previous_id+ assert not stored_id.startswith("error_")+ assert previous_id not in agent_module.active_conversations.get(user_id, {})+ assert stored_id in agent_module.active_conversations.get(user_id, {})+++async def test_langflow_stream_treats_credential_dump_content_as_error(+ monkeypatch, store_in_memory+):+ """WatsonX failures are often streamed as assistant text with no exception."""+ dump = (+ "Failed to initialize IBM WatsonX embedding model: Attempt of authenticating "+ "connection to service failed, please validate your credentials. Error: "+ '{"errorCode":"BXNIM0415E","errorMessage":"Provided API key could not be found."} '+ "IBM WatsonX requires additional configuration parameters "+ "(API endpoint URL and project ID). An error occurred while generating a response."+ )++ async def stream_dump(*_args, **_kwargs) -> AsyncIterator[bytes]:+ yield (json.dumps({"id": "resp_dump", "delta": {"content": dump}}) + "\n").encode("utf-8")++ monkeypatch.setattr(agent_module, "async_stream", stream_dump)++ user_id = "test-provider-error-content-dump"+ agent_module.active_conversations.pop(user_id, None)++ chunks = await _collect_error_chunks(+ agent_module.async_langflow_chat_stream(+ langflow_client=object(),+ flow_id="flow-id",+ prompt="hello",+ user_id=user_id,+ )+ )++ assert any(c.get("status") == "failed" for c in chunks)+ error_chunk = next(c for c in chunks if c.get("status") == "failed")+ assert "{" not in error_chunk["error"]["message"]+ assert "Provided API key could not be found" in error_chunk["error"]["message"]+ assert store_in_memory["stored"] == [(user_id, "resp_dump")]+ assert store_in_memory["claimed"] == [(user_id, "resp_dump")]+ stored = agent_module.active_conversations[user_id]["resp_dump"]+ assert stored["messages"][-1]["error"] is True+ assert "{" not in stored["messages"][-1]["content"]+++async def test_langflow_stream_keeps_partial_text_with_provider_error(monkeypatch, store_in_memory):+ provider_message = "Rate limit exceeded for watsonx.ai."++ async def partial_then_fail(*_args, **_kwargs) -> AsyncIterator[bytes]:+ yield (+ json.dumps(+ {+ "id": "resp_partial",+ "delta": {"content": "Here is a partial answer"},+ }+ )+ + "\n"+ ).encode("utf-8")+ raise Exception(provider_message)++ monkeypatch.setattr(agent_module, "async_stream", partial_then_fail)++ user_id = "test-provider-error-partial"+ agent_module.active_conversations.pop(user_id, None)++ chunks = await _collect_error_chunks(+ agent_module.async_langflow_chat_stream(+ langflow_client=object(),+ flow_id="flow-id",+ prompt="hello",+ user_id=user_id,+ )+ )++ assert len(chunks) == 2+ assert chunks[0]["delta"]["content"] == "Here is a partial answer"+ assert chunks[1]["error"]["message"] == (f"Here is a partial answer\n\n{provider_message}")+ assert store_in_memory["stored"] == [(user_id, "resp_partial")]+ assert store_in_memory["claimed"] == [(user_id, "resp_partial")]+++async def test_chat_stream_yields_provider_error_chunk(monkeypatch, store_in_memory):+ provider_message = "Permission denied: model access not authorized for this project."++ async def raise_provider_error(*_args, **_kwargs) -> AsyncIterator[bytes]:+ raise Exception(provider_message)+ yield b"" # pragma: no cover++ monkeypatch.setattr(agent_module, "async_stream", raise_provider_error)++ user_id = "test-chat-provider-error-user"+ agent_module.active_conversations.pop(user_id, None)++ chunks = await _collect_error_chunks(+ agent_module.async_chat_stream(+ async_client=object(),+ prompt="hello",+ user_id=user_id,+ model="gpt-4.1-mini",+ )+ )++ assert len(chunks) == 1+ assert chunks[0]["error"]["message"] == provider_message+ assert len(store_in_memory["stored"]) == 1+ stored_user, stored_id = store_in_memory["stored"][0]+ assert stored_user == user_id+ assert not stored_id.startswith("error_")+ assert chunks[0].get("id") == stored_id+ assert stored_id in agent_module.active_conversations.get(user_id, {})diff --git a/tests/unit/test_provider_error_formatting.py b/tests/unit/test_provider_error_formatting.pynew file mode 100644index 000000000..4a62f3716--- /dev/null+++ b/tests/unit/test_provider_error_formatting.py@@ -0,0 +1,171 @@+"""Unit tests for provider error message formatting helpers."""++from __future__ import annotations++import json++import pytest++from api.provider_validation import (+ format_provider_error_message,+ is_generic_upstream_error,+ is_provider_credential_error,+ looks_like_provider_error_content,+ resolve_chat_stream_error_message,+ sanitize_provider_error_content,+)+++def test_parse_ibm_iam_error_message():+ raw = (+ 'Failed to authenticate with IBM Watson: {"errorCode":"BXNIM0415E",'+ '"errorMessage":"Provided API key could not be found.",'+ '"context":{"requestId":"abc","url":"https://iam.cloud.ibm.com"}}'+ )+ assert format_provider_error_message(raw) == "Provided API key could not be found."+++def test_format_extracts_json_with_trailing_text():+ raw = (+ "Failed to initialize IBM WatsonX embedding model: Attempt of authenticating "+ "connection to service failed, please validate your credentials. Error: "+ '{"errorCode":"BXNIM0415E","errorMessage":"Provided API key could not be found.",'+ '"context":{"requestId":"abc"}} '+ "IBM WatsonX requires additional configuration parameters. "+ "An error occurred while generating a response."+ )+ assert format_provider_error_message(raw) == "Provided API key could not be found."+ assert "{" not in sanitize_provider_error_content(raw)+++def test_is_provider_credential_error():+ assert is_provider_credential_error("Incorrect API key provided")+ assert is_provider_credential_error("Provided API key could not be found.")+ assert is_provider_credential_error("Provided API key is disabled.")+ assert is_provider_credential_error(json.dumps({"errorMessage": "api key revoked"}))+ assert not is_provider_credential_error("Rate limit exceeded")+++def test_disabled_watsonx_key_embedding_dump_is_credential_error():+ """Langflow embedding failures for disabled keys omit 'failed to authenticate'."""+ raw = (+ "Error running graph: Error building Component Embedding Model: "+ "Failed to initialize IBM WatsonX embedding model: Attempt of authenticating "+ "connection to service failed, please validate your credentials. Error: "+ '{"errorCode":"BXNIM0420E","errorMessage":"Provided API key is disabled."}'+ )+ cleaned = sanitize_provider_error_content(raw)+ assert is_provider_credential_error(raw) or is_provider_credential_error(cleaned)+ assert cleaned == "Provided API key is disabled."+ assert "{" not in cleaned+++def test_strip_error_label_prefixes_without_json():+ assert (+ format_provider_error_message(+ "Error running graph: Error building Component Language Model: Rate limit exceeded"+ )+ == "Rate limit exceeded"+ )+ # Bare "Error: …" is preserved (no label after Error).+ assert format_provider_error_message("Error: boom") == "Error: boom"+++def test_looks_like_provider_error_content():+ assert looks_like_provider_error_content("Error: boom")+ assert looks_like_provider_error_content("An unknown error occurred.")+ assert looks_like_provider_error_content(+ 'Failed to authenticate: {"errorMessage":"Provided API key could not be found."}'+ )+ assert not looks_like_provider_error_content("OpenRAG is an open-source package.")+ assert is_generic_upstream_error("An unknown error occurred.")+++def test_resolve_chat_stream_error_message_keeps_generic_without_probing():+ # Opaque upstream text must not be rewritten into an inferred API-key error.+ assert (+ resolve_chat_stream_error_message("An unknown error occurred.")+ == "An unknown error occurred."+ )+++async def test_resolve_ingest_error_message_probes_credentials_on_disconnect(monkeypatch):+ async def fake_probe():+ return "Provided API key could not be found."++ monkeypatch.setattr(+ "api.provider_validation.probe_provider_credential_error",+ fake_probe,+ )++ from api.provider_validation import resolve_ingest_error_message++ assert (+ await resolve_ingest_error_message("Server disconnected without sending a response.")+ == "Provided API key could not be found."+ )+++async def test_resolve_ingest_error_message_keeps_disconnect_when_probe_clean(monkeypatch):+ async def fake_probe():+ return None++ monkeypatch.setattr(+ "api.provider_validation.probe_provider_credential_error",+ fake_probe,+ )++ from api.provider_validation import resolve_ingest_error_message++ raw = "Server disconnected without sending a response."+ assert await resolve_ingest_error_message(raw) == raw+++async def test_probe_checks_non_selected_providers_with_keys(monkeypatch):+ """Revoked watsonx must be detected even when openai is the selected provider."""++ class FakeProvider:+ def __init__(self, api_key="", endpoint=None, project_id=None):+ self.api_key = api_key+ self.endpoint = endpoint+ self.project_id = project_id++ class FakeConfig:+ class knowledge:+ embedding_provider = "openai"+ embedding_model = "text-embedding-3-small"++ class agent:+ llm_provider = "openai"+ llm_model = "gpt-4o-mini"++ class providers:+ openai = FakeProvider(api_key="sk-ok")+ anthropic = FakeProvider()+ watsonx = FakeProvider(api_key="bad-watsonx", project_id="proj")+ ollama = FakeProvider()++ def get_embedding_provider_config(self):+ return self.providers.openai++ def get_llm_provider_config(self):+ return self.providers.openai++ async def fake_validate(**kwargs):+ if kwargs.get("provider") == "watsonx":+ raise Exception(+ "Failed to authenticate with IBM Watson: Provided API key could not be found."+ )++ monkeypatch.setattr("config.settings.get_openrag_config", lambda: FakeConfig())+ monkeypatch.setattr("api.provider_validation.validate_provider_setup", fake_validate)++ from api.provider_validation import probe_provider_credential_error++ assert (+ await probe_provider_credential_error()+ == "Failed to authenticate with IBM Watson: Provided API key could not be found."+ )diff --git a/tests/unit/test_task_service_get_task_status2.py b/tests/unit/test_task_service_get_task_status2.pyindex 64660959f..cbb76008a 100644--- a/tests/unit/test_task_service_get_task_status2.py+++ b/tests/unit/test_task_service_get_task_status2.py@@ -170,6 +170,58 @@ class TestInferFailureMetadata:assert meta["failure_phase"] == "unknown"assert meta["actionable_by"] == "RETRYABLE"+ def test_langflow_revoked_api_key_failure(self, task_service):+ ft = _make_file_task(+ phase=IngestionPhase.LANGFLOW,+ docling_status=DoclingPhaseStatus.SUCCESS,+ error=(+ 'Failed to authenticate with IBM Watson: {"errorCode":"BXNIM0415E",'+ '"errorMessage":"Provided API key could not be found."}'+ ),+ )+ meta = task_service._infer_failure_metadata(ft)+ assert meta is not None+ assert meta["component"] == "openrag"+ assert meta["failure_phase"] == "embedding"+ assert meta["actionable_by"] == "USER_ACTIONABLE"+ assert "API key" in meta["user_facing_message"]+ assert "{" not in meta["user_facing_message"]++ def test_langflow_disabled_api_key_failure(self, task_service):+ """Disabled (not deleted) IBM keys must not fall through to generic Langflow copy."""+ ft = _make_file_task(+ phase=IngestionPhase.LANGFLOW,+ docling_status=DoclingPhaseStatus.SUCCESS,+ error=(+ "Error building Component Embedding Model: Failed to initialize IBM "+ "WatsonX embedding model: Attempt of authenticating connection to "+ "service failed, please validate your credentials. Error: "+ '{"errorCode":"BXNIM0420E","errorMessage":"Provided API key is disabled."}'+ ),+ )+ meta = task_service._infer_failure_metadata(ft)+ assert meta is not None+ assert meta["component"] == "openrag"+ assert meta["failure_phase"] == "embedding"+ assert meta["actionable_by"] == "USER_ACTIONABLE"+ assert "disabled" in meta["user_facing_message"].lower()+ assert "unexpectedly" not in meta["user_facing_message"].lower()+ assert "error building component" not in meta["user_facing_message"].lower()+ assert "error running graph" not in meta["user_facing_message"].lower()+ assert "{" not in meta["user_facing_message"]++ def test_langflow_disconnect_with_api_key_text_prefers_credentials(self, task_service):+ ft = _make_file_task(+ phase=IngestionPhase.LANGFLOW,+ docling_status=DoclingPhaseStatus.SUCCESS,+ error=("Server disconnected without sending a response. Incorrect API key provided"),+ )+ meta = task_service._infer_failure_metadata(ft)+ assert meta is not None+ assert meta["failure_phase"] == "embedding"+ assert "API key" in meta["user_facing_message"]+ assert "connection was lost" not in meta["user_facing_message"].lower()+def test_duplicate_file_error(self, task_service):ft = _make_file_task(phase=IngestionPhase.DOCLING,