Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: surface provider errors in chat instead of generic connection failures (#2127)

Wallgauac66bf0ecf52
diff --git a/Makefile b/Makefile
index b95465144..394319e32 100644
--- a/Makefile
+++ b/Makefile
@@ -736,11 +736,11 @@ frontend: ## Run frontend locally
echo "$(YELLOW)Detected Power architecture ($$ARCH) — using Webpack fallback$(NC)"; \
npx next dev --webpack \
--port $$PORT \
- --hostname $(hostname); \
+ --hostname $$(hostname); \
else \
npx next dev \
--port $$PORT \
- --hostname $(hostname); \
+ --hostname $$(hostname); \
fi
docling: ## Start docling-serve for document processing
diff --git a/frontend/app/chat/page.tsx b/frontend/app/chat/page.tsx
index 1e4f95195..759382c6b 100644
--- a/frontend/app/chat/page.tsx
+++ b/frontend/app/chat/page.tsx
@@ -11,6 +11,11 @@ import { useTask } from "@/contexts/task-context";
import { useOnboardingState } from "@/hooks/use-onboarding-state";
import { useChatStreaming } from "@/hooks/useChatStreaming";
import { trackLLMCall } from "@/lib/analytics";
+import {
+ dedupeConsecutiveErrorMessages,
+ formatProviderErrorMessage,
+ looksLikeProviderErrorContent,
+} from "@/lib/chat-stream-errors";
import { FILE_CONFIRMATION, FILES_REGEX } from "@/lib/constants";
import { buildSearchPayloadFilters } from "@/lib/filter-normalization";
import { uploadFileForContext } from "@/lib/upload-utils";
@@ -76,6 +81,9 @@ function ChatPage() {
const { scrollToBottom } = useStickToBottomContext();
const lastLoadedConversationRef = useRef<string | null>(null);
+ // Set when a live stream fails so history sync cannot replace one error card
+ // with Langflow's duplicated copies of the same failure.
+ const liveErrorConversationRef = useRef<string | null>(null);
const { addTask } = useTask();
// Check if chat history is loading
@@ -111,17 +119,57 @@ function ChatPage() {
} = useChatStreaming({
endpoint: apiEndpoint,
onComplete: (message, responseId) => {
+ setLoading(false);
+ setWaitingTooLong(false);
+
+ setMessages((prev) => {
+ if (!message.error) {
+ return [...prev, message];
+ }
+ // One error card per failure — drop a trailing duplicate if present.
+ const withoutTrailingDup = [...prev];
+ while (
+ withoutTrailingDup.length > 0 &&
+ withoutTrailingDup[withoutTrailingDup.length - 1]?.role ===
+ "assistant" &&
+ withoutTrailingDup[withoutTrailingDup.length - 1]?.error &&
+ withoutTrailingDup[withoutTrailingDup.length - 1]?.content ===
+ message.content
+ ) {
+ withoutTrailingDup.pop();
+ }
+ return [...withoutTrailingDup, message];
+ });
+
+ if (message.error) {
+ // Sidebar id stays on currentConversationId; onError clears Langflow chaining.
+ if (responseId) {
+ liveErrorConversationRef.current = responseId;
+ if (!currentConversationId) {
+ setCurrentConversationId(responseId);
+ refreshConversations(true);
+ if (conversationFilter && typeof window !== "undefined") {
+ localStorage.setItem(
+ `conversation_filter_${responseId}`,
+ conversationFilter.id,
+ );
+ }
+ } else {
+ refreshConversationsSilent();
+ }
+ }
+ return;
+ }
+
trackLLMCall({
mode: "chat",
model: settings?.agent?.llm_model,
inputTokens: message.usage?.input_tokens,
outputTokens: message.usage?.output_tokens,
});
- setMessages((prev) => [...prev, message]);
- setLoading(false);
- setWaitingTooLong(false);
if (responseId) {
cancelNudges();
+ // Langflow session id for chaining; sidebar id stays on currentConversationId.
setPreviousResponseIds((prev) => ({
...prev,
[endpoint]: responseId,
@@ -136,7 +184,8 @@ function ChatPage() {
// Save filter association for this response
if (conversationFilter && typeof window !== "undefined") {
- const newKey = `conversation_filter_${responseId}`;
+ const stableId = currentConversationId || responseId;
+ const newKey = `conversation_filter_${stableId}`;
localStorage.setItem(newKey, conversationFilter.id);
}
}
@@ -145,15 +194,13 @@ function ChatPage() {
console.error("Streaming error:", error);
setLoading(false);
setWaitingTooLong(false);
- // Set chat error flag to trigger test_completion=true on health checks
+ // Set chat error flag to trigger test_completion=true on health checks.
setChatError(true);
- const errorMessage: Message = {
- role: "assistant",
- content:
- "Sorry, I couldn't connect to the chat service. Please try again.",
- timestamp: new Date(),
- };
- setMessages((prev) => [...prev, errorMessage]);
+ // Clear Langflow session chaining; conversation_id keeps the sidebar thread.
+ setPreviousResponseIds((prev) => ({
+ ...prev,
+ [endpoint]: null,
+ }));
},
});
@@ -269,6 +316,7 @@ function ChatPage() {
setIsFilterHighlighted(false);
setLoading(false);
lastLoadedConversationRef.current = null;
+ liveErrorConversationRef.current = null;
// Focus input after a short delay to ensure rendering is complete
setTimeout(() => {
@@ -291,20 +339,47 @@ function ChatPage() {
// Load conversation data from context
useEffect(() => {
let focusTimeoutId: NodeJS.Timeout;
- // Only load conversation data when:
- // 1. conversationData exists AND
- // 2. (It's a different conversation OR we're not streaming and data has changed) AND
- // 3. User is not in the middle of an interaction
- const isNewConversation =
- lastLoadedConversationRef.current !== conversationData?.response_id;
- const hasMessageCountChanged =
- conversationData?.messages?.length !== messages.length;
-
- if (
- conversationData?.messages &&
- (isNewConversation || (!isChatStreaming && hasMessageCountChanged)) &&
+ // Only load conversation data when remote history should win:
+ // - Switching to a different conversation always loads remote.
+ // - Same conversation: never clobber local turns that are still ahead
+ // (failed sends append user+error before history refreshes; syncing
+ // stale remote would wipe them and retries then duplicate in Langflow).
+ const conversationId = conversationData?.response_id ?? null;
+ const isSwitchingConversation =
+ conversationId != null &&
+ lastLoadedConversationRef.current != null &&
+ lastLoadedConversationRef.current !== conversationId;
+ const isFirstLoadOfConversation =
+ conversationId != null &&
+ lastLoadedConversationRef.current !== conversationId;
+ const remoteMessageCount = conversationData?.messages?.length ?? 0;
+ const hasMessageCountChanged = remoteMessageCount !== messages.length;
+ const localMessagesAhead = messages.length > remoteMessageCount;
+ // After a live failed send we already appended the error locally. History
+ // often returns the same provider failure repeated from Langflow — do not
+ // clobber the live transcript for that conversation id.
+ const skipSyncAfterLiveError =
+ conversationId != null &&
+ liveErrorConversationRef.current === conversationId &&
+ !isSwitchingConversation;
+ const shouldSyncSameConversation =
+ !isChatStreaming &&
+ hasMessageCountChanged &&
+ !localMessagesAhead &&
!isUserInteracting &&
- !isForkingInProgress
+ !isForkingInProgress;
+
+ if (skipSyncAfterLiveError) {
+ lastLoadedConversationRef.current = conversationId;
+ setPreviousResponseIds((prev) => ({
+ ...prev,
+ [conversationData?.endpoint ?? endpoint]: null,
+ }));
+ } else if (
+ conversationData?.messages &&
+ (isSwitchingConversation ||
+ (isFirstLoadOfConversation && !localMessagesAhead) ||
+ (!isFirstLoadOfConversation && shouldSyncSameConversation))
) {
// Convert backend message format to frontend Message interface
const convertedMessages: Message[] = conversationData.messages.map(
@@ -337,11 +412,17 @@ function ChatPage() {
}>;
response_data?: unknown;
}) => {
+ const isProviderError =
+ Boolean(msg.error) ||
+ (msg.role === "assistant" &&
+ looksLikeProviderErrorContent(msg.content || ""));
const message: Message = {
role: msg.role as "user" | "assistant",
- content: msg.content,
+ content: isProviderError
+ ? formatProviderErrorMessage(msg.content)
+ : msg.content,
timestamp: new Date(msg.timestamp || new Date()),
- error: msg.error || false,
+ error: isProviderError,
};
// Extract function calls from chunks or response_data
@@ -458,13 +539,21 @@ function ChatPage() {
},
);
- setMessages(convertedMessages);
+ const dedupedMessages = dedupeConsecutiveErrorMessages(convertedMessages);
+ setMessages(dedupedMessages);
lastLoadedConversationRef.current = conversationData.response_id;
+ if (liveErrorConversationRef.current === conversationData.response_id) {
+ liveErrorConversationRef.current = null;
+ }
- // Set the previous response ID for this conversation
+ // Don't chain a session that ended in an error — Langflow often collapses
+ // follow-ups to "An unknown error occurred." and hides the real failure.
+ const lastConverted = dedupedMessages[dedupedMessages.length - 1];
setPreviousResponseIds((prev) => ({
...prev,
- [conversationData.endpoint]: conversationData.response_id,
+ [conversationData.endpoint]: lastConverted?.error
+ ? null
+ : conversationData.response_id,
}));
// Focus input when loading a conversation
@@ -484,6 +573,7 @@ function ChatPage() {
setPreviousResponseIds,
isChatStreaming,
messages.length,
+ endpoint,
]);
// Handle new conversation creation - only reset messages when placeholderConversation is set
@@ -533,13 +623,21 @@ function ChatPage() {
})()
: undefined;
- // Use passed previousResponseId if available, otherwise fall back to state
- const responseIdToUse = previousResponseId || previousResponseIds[endpoint];
+ // OpenRAG sidebar thread vs Langflow session are separate:
+ // - conversationId keeps the same list entry after errors
+ // - previousResponseId is omitted after an error so Langflow starts fresh
+ const lastAssistant = [...messages]
+ .reverse()
+ .find((message) => message.role === "assistant");
+ const langflowSessionId = lastAssistant?.error
+ ? undefined
+ : previousResponseId || previousResponseIds[endpoint] || undefined;
// Use the hook to send the message
await sendStreamingMessage({
prompt: userMessage.content,
- previousResponseId: responseIdToUse || undefined,
+ previousResponseId: langflowSessionId,
+ conversationId: currentConversationId || undefined,
filters: processedFilters,
filter_id: conversationFilter?.id, // ✅ Add filter_id for this conversation
limit: parsedFilterData?.limit ?? 10,
diff --git a/frontend/app/onboarding/_components/onboarding-content.tsx b/frontend/app/onboarding/_components/onboarding-content.tsx
index 97f0a645d..7f0362f68 100644
--- a/frontend/app/onboarding/_components/onboarding-content.tsx
+++ b/frontend/app/onboarding/_components/onboarding-content.tsx
@@ -148,13 +148,18 @@ export function OnboardingContent({
const { streamingMessage, isLoading, sendMessage } = useChatStreaming({
onComplete: async (message, newResponseId) => {
+ setAssistantMessage(message);
+ // Errors are display-only during onboarding — do not track or persist them.
+ if (message.error) {
+ return;
+ }
+
trackLLMCall({
mode: "onboarding",
model: settings?.agent?.llm_model,
inputTokens: message.usage?.input_tokens,
outputTokens: message.usage?.output_tokens,
});
- setAssistantMessage(message);
// Save assistant message to backend
await updateOnboardingMutation.mutateAsync({
assistant_message: {
@@ -194,11 +199,19 @@ export function OnboardingContent({
},
onError: (error) => {
console.error("Chat error:", error);
- setAssistantMessage({
- role: "assistant",
- content:
- "Sorry, I couldn't connect to the chat service. Please try again.",
- timestamp: new Date(),
+ // Display is owned by onComplete (including message.error). Keep a
+ // fallback only when completion never ran.
+ setAssistantMessage((prev) => {
+ if (prev?.error || (prev && !prev.isStreaming)) {
+ return prev;
+ }
+ return {
+ role: "assistant",
+ content:
+ error.message || "An error occurred while generating a response.",
+ timestamp: new Date(),
+ error: true,
+ };
});
},
});
diff --git a/frontend/components/provider-health-banner.tsx b/frontend/components/provider-health-banner.tsx
index 722cb8ed9..faa345c10 100644
--- a/frontend/components/provider-health-banner.tsx
+++ b/frontend/components/provider-health-banner.tsx
@@ -73,26 +73,27 @@ export function ProviderHealthBanner({ className }: ProviderHealthBannerProps) {
let errorProvider: string | undefined;
let errorMessage: string;
+ // Prefer a single shared provider when LLM and embedding fail the same way
+ // (e.g. both watsonx auth failures), so the banner stays readable.
+ let showMultipleErrors = false;
+
if (llmError && embeddingError) {
- // Both have errors - check if they're the same
if (llmError === embeddingError) {
- // Same error for both - show once
errorMessage = llmError;
+ errorProvider =
+ llmProvider === embeddingProvider ? llmProvider : undefined;
} else {
- // Different errors - show both
errorMessage = `${llmError}; ${embeddingError}`;
+ errorProvider = undefined;
+ showMultipleErrors = true;
}
- errorProvider = undefined; // Don't link to a specific provider
} else if (llmError) {
- // Only LLM has error
errorProvider = llmProvider;
errorMessage = llmError;
} else if (embeddingError) {
- // Only embedding has error
errorProvider = embeddingProvider;
errorMessage = embeddingError;
} else {
- // Fallback to original message
errorMessage = health?.message || "Provider validation failed";
errorProvider = llmProvider;
}
@@ -105,6 +106,10 @@ export function ProviderHealthBanner({ className }: ProviderHealthBannerProps) {
? `/settings?setup=${errorProvider}`
: "/settings";
+ const bannerLabel = showMultipleErrors
+ ? `Provider errors - ${errorMessage}`
+ : `${providerTitle} error - ${errorMessage}`;
+
return (
<Banner
className={cn(
@@ -117,13 +122,7 @@ export function ProviderHealthBanner({ className }: ProviderHealthBannerProps) {
icon={AlertTriangle}
/>
<BannerTitle className="font-medium flex items-center gap-2">
- {llmError && embeddingError ? (
- <>Provider errors - {errorMessage}</>
- ) : (
- <>
- {providerTitle} error - {errorMessage}
- </>
- )}
+ {bannerLabel}
</BannerTitle>
<Button size="sm" onClick={() => router.push(settingsUrl)}>
Fix Setup
diff --git a/frontend/contexts/task-context.tsx b/frontend/contexts/task-context.tsx
index 93ce5b78b..981f5f6f0 100644
--- a/frontend/contexts/task-context.tsx
+++ b/frontend/contexts/task-context.tsx
@@ -28,6 +28,7 @@ import {
getKnowledgeFileIdentity,
inferTaskFileConnectorType,
} from "@/lib/knowledge-table-state";
+import { getTaskFailureToastDescription } from "@/lib/task-error-display";
import {
didTaskReachCompleted,
didTaskReachTerminalState,
@@ -496,9 +497,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
};
if (isTotalFailure) {
toast.error("Task failed", {
- description: `${failedFiles} file${
- failedFiles !== 1 ? "s" : ""
- } failed`,
+ description: getTaskFailureToastDescription(currentTask),
action: toastAction,
});
} else {
@@ -626,9 +625,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
resultValue: currentTask.error,
});
toast.error("Task failed", {
- description: `Task ${currentTask.task_id} failed: ${
- currentTask.error || "Unknown error"
- }`,
+ description: getTaskFailureToastDescription(currentTask),
});
// Set chat error flag to trigger test_completion=true on health checks
diff --git a/frontend/hooks/useChatStreaming.ts b/frontend/hooks/useChatStreaming.ts
index 2644846b3..761a23ce5 100644
--- a/frontend/hooks/useChatStreaming.ts
+++ b/frontend/hooks/useChatStreaming.ts
@@ -5,6 +5,11 @@ import type {
TokenUsage,
} from "@/app/chat/_types/types";
import { useChat } from "@/contexts/chat-context";
+import {
+ extractStreamProviderError,
+ formatProviderErrorMessage,
+ looksLikeProviderErrorContent,
+} from "@/lib/chat-stream-errors";
import {
detectImplicitToolCall,
detectRAGFromContent,
@@ -24,6 +29,8 @@ interface UseChatStreamingOptions {
interface SendMessageOptions {
prompt: string;
previousResponseId?: string;
+ /** OpenRAG sidebar thread id (kept after errors; distinct from Langflow session). */
+ conversationId?: string;
filters?: FilterInput;
filter_id?: string;
limit?: number;
@@ -47,6 +54,7 @@ export function useChatStreaming({
const sendMessage = async ({
prompt,
previousResponseId,
+ conversationId,
filters,
filter_id,
limit = 10,
@@ -55,6 +63,8 @@ export function useChatStreaming({
// Set up timeout to detect stuck/hanging requests
let timeoutId: NodeJS.Timeout | null = null;
let hasReceivedData = false;
+ // Hoisted so the catch path can still attach a failed turn to history.
+ let newResponseId: string | null = null;
try {
setIsLoading(true);
@@ -86,6 +96,7 @@ export function useChatStreaming({
prompt: string;
stream: boolean;
previous_response_id?: string;
+ conversation_id?: string;
filters?: FilterInput;
filter_id?: string;
limit?: number;
@@ -101,6 +112,10 @@ export function useChatStreaming({
requestBody.previous_response_id = previousResponseId;
}
+ if (conversationId) {
+ requestBody.conversation_id = conversationId;
+ }
+
if (filters) {
const payloadFilters = buildSearchPayloadFilters(filters);
if (payloadFilters) {
@@ -139,9 +154,8 @@ export function useChatStreaming({
let buffer = "";
const content = { value: "" };
const currentFunctionCalls: FunctionCall[] = [];
- let newResponseId: string | null = null;
- let isError = false;
const usage: { value: TokenUsage | undefined } = { value: undefined };
+ let providerStreamError: string | null = null;
if (!controller.signal.aborted && thisStreamId === streamIdRef.current) {
setStreamingMessage({
@@ -190,12 +204,10 @@ export function useChatStreaming({
parseOpenRAGChunk(chunk, content);
detectImplicitToolCall(chunk, currentFunctionCalls);
- if (
- chunk.finish_reason === "error" ||
- chunk.status === "failed"
- ) {
- console.error("Error detected in stream");
- isError = true;
+ const streamError = extractStreamProviderError(chunk);
+ if (streamError) {
+ console.error("Error detected in stream", streamError);
+ providerStreamError = streamError;
break streamLoop;
}
@@ -225,6 +237,21 @@ export function useChatStreaming({
if (timeoutId) clearTimeout(timeoutId);
}
+ // Prefer accumulated stream text when it carries the real provider dump —
+ // Langflow often sends finish_reason=error with an empty error payload.
+ if (content.value && looksLikeProviderErrorContent(content.value)) {
+ throw Object.assign(
+ new Error(formatProviderErrorMessage(content.value)),
+ { partialContent: content.value },
+ );
+ }
+
+ if (providerStreamError) {
+ throw Object.assign(new Error(providerStreamError), {
+ partialContent: content.value,
+ });
+ }
+
if (
!hasReceivedData ||
(!content.value && currentFunctionCalls.length === 0)
@@ -246,7 +273,6 @@ export function useChatStreaming({
currentFunctionCalls.length > 0 ? currentFunctionCalls : undefined,
timestamp: new Date(),
isStreaming: false,
- error: isError,
usage: usage.value,
};
@@ -276,7 +302,9 @@ export function useChatStreaming({
// Create user-friendly error message
const errorMessage = (error as Error).message;
- let errorContent = errorMessage; // Default to the actual error message
+ let errorContent = formatProviderErrorMessage(
+ errorMessage || "An error occurred while generating a response.",
+ );
// Only override with generic messages for specific infrastructure errors
if (errorMessage?.includes("timed out")) {
@@ -291,9 +319,30 @@ export function useChatStreaming({
errorContent =
"Network error. Please check your connection and try again.";
}
- // For all other errors (including Langflow errors), use the actual error message
- onError?.(error as Error);
+ // Keep any mid-stream partial answer visible above the provider error,
+ // unless the "partial" is itself a provider failure dump with JSON.
+ const partialContent =
+ typeof (error as { partialContent?: unknown }).partialContent ===
+ "string"
+ ? (error as { partialContent: string }).partialContent.trim()
+ : "";
+ const partialLooksLikeProviderError =
+ partialContent.includes("{") ||
+ /api key|authenticat|unauthorized|permission denied|rate limit/i.test(
+ partialContent,
+ );
+ if (
+ partialContent &&
+ !partialLooksLikeProviderError &&
+ !errorContent.startsWith(partialContent) &&
+ !errorMessage?.includes("timed out") &&
+ !errorMessage?.includes("No response") &&
+ !errorMessage?.includes("NetworkError") &&
+ !errorMessage?.includes("Failed to fetch")
+ ) {
+ errorContent = `${partialContent}\n\n${errorContent}`;
+ }
const errorMessageObj: Message = {
role: "assistant",
@@ -303,10 +352,19 @@ export function useChatStreaming({
error: true,
};
- // Pass error message to onComplete so it gets added to chat history
- // This ensures errors appear immediately and persist on page refresh
- if (!streamAbortRef.current?.signal.aborted) {
- onComplete?.(errorMessageObj, null);
+ // onError owns side effects (flags, session reset). onComplete owns appending
+ // the error message and refreshing history — pass any stream/store id so a
+ // failed first turn still appears in the conversation list.
+ onError?.(
+ Object.assign(new Error(errorContent), {
+ partialContent,
+ }) as Error,
+ );
+ // User aborts skip completion; timeout aborts still surface the error card.
+ const isTimeout = errorMessage?.includes("timed out");
+ if (!streamAbortRef.current?.signal.aborted || isTimeout) {
+ onComplete?.(errorMessageObj, newResponseId);
+ refreshConversations(true);
}
return errorMessageObj;
diff --git a/frontend/lib/chat-stream-errors.test.ts b/frontend/lib/chat-stream-errors.test.ts
new file mode 100644
index 000000000..7a37dc758
--- /dev/null
+++ b/frontend/lib/chat-stream-errors.test.ts
@@ -0,0 +1,175 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import {
+ dedupeConsecutiveErrorMessages,
+ extractStreamProviderError,
+ formatProviderErrorMessage,
+ looksLikeProviderErrorContent,
+} from "./chat-stream-errors";
+
+describe("extractStreamProviderError", () => {
+ it("returns null for non-error chunks", () => {
+ assert.equal(extractStreamProviderError({ status: "ok" }), null);
+ assert.equal(extractStreamProviderError(null), null);
+ });
+
+ it("reads error.message from failed provider chunks", () => {
+ assert.equal(
+ extractStreamProviderError({
+ status: "failed",
+ finish_reason: "error",
+ error: {
+ message:
+ "Rate limit exceeded for watsonx.ai. Please try again later.",
+ },
+ }),
+ "Rate limit exceeded for watsonx.ai. Please try again later.",
+ );
+ });
+
+ it("reads string error and top-level message fallbacks", () => {
+ assert.equal(
+ extractStreamProviderError({
+ status: "failed",
+ error: "Invalid API key for Anthropic.",
+ }),
+ "Invalid API key for Anthropic.",
+ );
+ assert.equal(
+ extractStreamProviderError({
+ finish_reason: "error",
+ message: "Permission denied: model access not authorized.",
+ }),
+ "Permission denied: model access not authorized.",
+ );
+ });
+
+ it("returns null when the error chunk has no message", () => {
+ assert.equal(extractStreamProviderError({ status: "failed" }), null);
+ assert.equal(extractStreamProviderError({ finish_reason: "error" }), null);
+ });
+
+ it("strips embedded JSON from provider error chunks", () => {
+ assert.equal(
+ extractStreamProviderError({
+ status: "failed",
+ error: {
+ message:
+ 'Failed to authenticate with IBM Watson: {"errorCode":"BXNIM0415E","errorMessage":"Provided API key could not be found."}',
+ },
+ }),
+ "Provided API key could not be found.",
+ );
+ });
+});
+
+describe("looksLikeProviderErrorContent", () => {
+ it("detects watsonx credential dumps streamed as assistant text", () => {
+ assert.equal(
+ looksLikeProviderErrorContent(
+ 'Failed to initialize IBM WatsonX embedding model: Error: {"errorCode":"BXNIM0415E","errorMessage":"Provided API key could not be found."} An error occurred while generating a response.',
+ ),
+ true,
+ );
+ });
+
+ it("detects permission-denied style provider errors", () => {
+ assert.equal(
+ looksLikeProviderErrorContent(
+ "Permission denied: model access not authorized.",
+ ),
+ true,
+ );
+ });
+
+ it("does not flag ordinary replies", () => {
+ assert.equal(
+ looksLikeProviderErrorContent("OpenRAG uses Langflow and OpenSearch."),
+ false,
+ );
+ assert.equal(
+ looksLikeProviderErrorContent(
+ "The docs explain unauthorized access patterns and when permission denied responses appear.",
+ ),
+ false,
+ );
+ });
+
+ it("flags IBM disabled API key messages", () => {
+ assert.equal(
+ looksLikeProviderErrorContent("Provided API key is disabled."),
+ true,
+ );
+ });
+
+ it("flags the exact generic stream fallback message", () => {
+ assert.equal(
+ looksLikeProviderErrorContent(
+ "An error occurred while generating a response.",
+ ),
+ true,
+ );
+ });
+});
+
+describe("dedupeConsecutiveErrorMessages", () => {
+ it("collapses repeated identical assistant errors", () => {
+ const err = {
+ role: "assistant",
+ content: "Provided API key could not be found.",
+ error: true,
+ };
+ assert.deepEqual(
+ dedupeConsecutiveErrorMessages([
+ { role: "user", content: "hello" },
+ err,
+ { ...err },
+ { ...err },
+ ]),
+ [{ role: "user", content: "hello" }, err],
+ );
+ });
+});
+
+describe("formatProviderErrorMessage", () => {
+ it("extracts OpenAI-style embedded JSON", () => {
+ assert.equal(
+ formatProviderErrorMessage(
+ 'Provider request failed: {"error":{"message":"Incorrect API key provided","type":"invalid_request_error"}}',
+ ),
+ "Incorrect API key provided",
+ );
+ });
+
+ it("strips embedded JSON even with trailing text", () => {
+ assert.equal(
+ formatProviderErrorMessage(
+ 'Failed to authenticate. Error: {"errorCode":"BXNIM0415E","errorMessage":"Provided API key could not be found."} trailing junk',
+ ),
+ "Provided API key could not be found.",
+ );
+ });
+
+ it("keeps a readable prefix when JSON is truncated", () => {
+ assert.equal(
+ formatProviderErrorMessage("Invalid API key {not-valid-json"),
+ "Invalid API key",
+ );
+ });
+
+ it("strips Error <label>: wrappers and prefers nested JSON messages", () => {
+ assert.equal(
+ formatProviderErrorMessage(
+ "Error running graph: Error building Component Language Model: Rate limit exceeded",
+ ),
+ "Rate limit exceeded",
+ );
+ assert.equal(
+ formatProviderErrorMessage(
+ 'Error running graph: Error building Component Embedding Model: Failed to authenticate. Error: {"errorCode":"BXNIM0420E","errorMessage":"Provided API key is disabled."}',
+ ),
+ "Provided API key is disabled.",
+ );
+ assert.equal(formatProviderErrorMessage("Error: boom"), "Error: boom");
+ });
+});
diff --git a/frontend/lib/chat-stream-errors.ts b/frontend/lib/chat-stream-errors.ts
new file mode 100644
index 000000000..af2b5671c
--- /dev/null
+++ b/frontend/lib/chat-stream-errors.ts
@@ -0,0 +1,286 @@
+/**
+ * Extract / sanitize provider stream errors for chat display.
+ */
+
+/**
+ * Peel leading "Error <label>:" wrappers (Langflow graph/component noise, etc.).
+ * Requires a label after "Error" so bare "Error: …" messages are preserved.
+ */
+const ERROR_LABEL_PREFIX = /^Error\s+\S[^:]*:\s*/i;
+
+function stripErrorLabelPrefixes(text: string): string {
+ let cleaned = text.trim();
+ while (true) {
+ const updated = cleaned.replace(ERROR_LABEL_PREFIX, "").trim();
+ if (updated === cleaned) {
+ return cleaned;
+ }
+ cleaned = updated;
+ }
+}
+
+function tryParseJsonMessage(text: string): string | null {
+ try {
+ const data = JSON.parse(text) as Record<string, unknown>;
+ if (!data || typeof data !== "object") {
+ return null;
+ }
+
+ const errors = data.errors;
+ if (Array.isArray(errors) && errors[0] && typeof errors[0] === "object") {
+ const first = errors[0] as { message?: unknown; code?: unknown };
+ if (typeof first.message === "string" && first.message.trim()) {
+ return first.message;
+ }
+ if (typeof first.code === "string" && first.code.trim()) {
+ return `Error: ${first.code}`;
+ }
+ }
+
+ const errorObj = data.error;
+ if (errorObj && typeof errorObj === "object") {
+ const message = (errorObj as { message?: unknown }).message;
+ if (typeof message === "string" && message.trim()) {
+ return message;
+ }
+ }
+
+ if (typeof data.message === "string" && data.message.trim()) {
+ return data.message;
+ }
+ if (typeof data.errorMessage === "string" && data.errorMessage.trim()) {
+ return data.errorMessage;
+ }
+ if (typeof data.detail === "string" && data.detail.trim()) {
+ return data.detail;
+ }
+ } catch {
+ return null;
+ }
+ return null;
+}
+
+function extractBalancedJsonObject(text: string): string | null {
+ const start = text.indexOf("{");
+ if (start < 0) {
+ return null;
+ }
+ let depth = 0;
+ let inString = false;
+ let escaped = false;
+ for (let i = start; i < text.length; i++) {
+ const ch = text[i];
+ if (inString) {
+ if (escaped) {
+ escaped = false;
+ } else if (ch === "\\") {
+ escaped = true;
+ } else if (ch === '"') {
+ inString = false;
+ }
+ continue;
+ }
+ if (ch === '"') {
+ inString = true;
+ } else if (ch === "{") {
+ depth += 1;
+ } else if (ch === "}") {
+ depth -= 1;
+ if (depth === 0) {
+ return text.slice(start, i + 1);
+ }
+ }
+ }
+ return null;
+}
+
+/**
+ * True when assistant text looks like a provider/auth failure rather than a reply.
+ * Marker list aligned with src/api/provider_validation.py (_PROVIDER_ERROR_CONTENT_MARKERS).
+ * Ambiguous phrases use tighter patterns so ordinary assistant prose is not flagged.
+ */
+export function looksLikeProviderErrorContent(text: string): boolean {
+ const trimmed = text.trim();
+ if (!trimmed) {
+ return false;
+ }
+ if (trimmed.startsWith("Error:")) {
+ return true;
+ }
+ const lowered = trimmed.toLowerCase().replace(/\.$/, "");
+ if (
+ lowered === "an unknown error occurred" ||
+ lowered === "an error occurred while generating a response" ||
+ lowered === "unknown error occurred" ||
+ lowered === "something went wrong"
+ ) {
+ return true;
+ }
+ const markers = [
+ "incorrect api key",
+ "invalid api key",
+ "invalid_api_key",
+ "api key could not be found",
+ "api key is invalid",
+ "api key has been revoked",
+ "api key revoked",
+ "revoked api key",
+ "provided api key could not be found",
+ // IBM IAM: key still exists but is toggled off (vs deleted/not found).
+ "api key is disabled",
+ "api key disabled",
+ "authentication_error",
+ "failed to authenticate",
+ "invalid x-api-key",
+ "authentication failed",
+ "invalid credentials",
+ "could not authenticate",
+ "rate limit",
+ "rate_limit",
+ "provider request failed",
+ "insufficient_quota",
+ ];
+ if (markers.some((marker) => lowered.includes(marker))) {
+ return true;
+ }
+ // Tighter matches for phrases that also appear in ordinary explanations.
+ if (
+ /(?:^|\berror\b[:\s]*|\b(?:401|403)\b[:\s]*)unauthorized\b/.test(lowered) ||
+ /(?:^|\berror\b[:\s]*)permission denied\b/.test(lowered) ||
+ /\bpermission denied\s*:/.test(lowered) ||
+ /(?:^|\berror\b[:\s]*)quota exceeded\b/.test(lowered) ||
+ /\bquota exceeded\s*:/.test(lowered)
+ ) {
+ return true;
+ }
+ if (
+ trimmed.includes("{") &&
+ (lowered.includes('"error"') ||
+ lowered.includes('"errormessage"') ||
+ lowered.includes('"errorcode"') ||
+ lowered.includes('"errors"'))
+ ) {
+ return true;
+ }
+ return false;
+}
+
+/**
+ * Strip embedded provider JSON payloads so chat never shows raw error objects.
+ */
+export function formatProviderErrorMessage(text: string): string {
+ const trimmed = stripErrorLabelPrefixes(text.trim());
+ if (!trimmed) {
+ return "An error occurred while generating a response.";
+ }
+
+ const direct = tryParseJsonMessage(trimmed);
+ if (direct) {
+ // Pure JSON payloads: prefer the extracted message only.
+ return stripErrorLabelPrefixes(direct);
+ }
+
+ const jsonBlob = extractBalancedJsonObject(trimmed);
+ if (jsonBlob) {
+ const nested = tryParseJsonMessage(jsonBlob);
+ if (nested) {
+ // Embedded provider JSON already has the real message — drop wrappers.
+ return stripErrorLabelPrefixes(nested);
+ }
+ const prefix = stripErrorLabelPrefixes(
+ trimmed
+ .slice(0, trimmed.indexOf(jsonBlob))
+ .replace(/[:\s]+$/, "")
+ .trim(),
+ );
+ if (prefix) {
+ return prefix;
+ }
+ }
+
+ // Truncated / unbalanced JSON — keep the readable prefix only.
+ const jsonStart = trimmed.indexOf("{");
+ if (jsonStart > 0) {
+ const prefix = stripErrorLabelPrefixes(
+ trimmed
+ .slice(0, jsonStart)
+ .replace(/[:\s]+$/, "")
+ .trim(),
+ );
+ if (prefix) {
+ return prefix;
+ }
+ }
+
+ return trimmed;
+}
+
+/**
+ * Collapse consecutive identical assistant error messages.
+ * Langflow often stores the same provider failure multiple times in one session.
+ */
+export function dedupeConsecutiveErrorMessages<
+ T extends { role?: string; content?: string; error?: boolean },
+>(messages: T[]): T[] {
+ const result: T[] = [];
+ for (const message of messages) {
+ const prev = result[result.length - 1];
+ const isErrorAssistant =
+ message.role === "assistant" &&
+ (Boolean(message.error) ||
+ looksLikeProviderErrorContent(message.content || ""));
+ const prevIsSameError =
+ prev?.role === "assistant" &&
+ (Boolean(prev.error) ||
+ looksLikeProviderErrorContent(prev.content || "")) &&
+ (prev.content || "").trim() === (message.content || "").trim();
+ if (isErrorAssistant && prevIsSameError) {
+ continue;
+ }
+ result.push(message);
+ }
+ return result;
+}
+
+/**
+ * Extract a user-facing provider/stream error message from an NDJSON chunk.
+ * Returns null when the chunk is not a failed/error terminal event.
+ */
+export function extractStreamProviderError(chunk: unknown): string | null {
+ if (!chunk || typeof chunk !== "object") {
+ return null;
+ }
+
+ const c = chunk as {
+ finish_reason?: unknown;
+ status?: unknown;
+ error?: unknown;
+ message?: unknown;
+ };
+
+ if (c.finish_reason !== "error" && c.status !== "failed") {
+ return null;
+ }
+
+ let raw: string | null = null;
+
+ if (typeof c.error === "string" && c.error.trim()) {
+ raw = c.error;
+ } else if (c.error && typeof c.error === "object") {
+ const message = (c.error as { message?: unknown }).message;
+ if (typeof message === "string" && message.trim()) {
+ raw = message;
+ }
+ } else if (typeof c.message === "string" && c.message.trim()) {
+ raw = c.message;
+ }
+
+ // Empty error payloads are common from Langflow (finish_reason=error with no
+ // message). Return null so the client can fall back to accumulated content or
+ // a later sanitized error chunk instead of a generic placeholder.
+ if (!raw) {
+ return null;
+ }
+
+ return formatProviderErrorMessage(raw);
+}
diff --git a/frontend/lib/task-error-display.ts b/frontend/lib/task-error-display.ts
index 87aa39406..b6b4c9a49 100644
--- a/frontend/lib/task-error-display.ts
+++ b/frontend/lib/task-error-display.ts
@@ -173,6 +173,37 @@ export function resolveTaskFileError(
return "Unknown error";
}
+/** Best available user-facing reason for a failed ingestion task toast. */
+export function getTaskFailureToastDescription(task: {
+ error?: string | null;
+ failed_files?: number;
+ files?: Record<string, TaskFileEntry> | null;
+}): string {
+ const files = Object.values(task.files ?? {});
+ for (const fileInfo of files) {
+ if (fileInfo.status !== "failed" && fileInfo.status !== "error") {
+ continue;
+ }
+ const message = resolveTaskFileError(fileInfo, task.error ?? undefined);
+ if (message && message !== "Unknown error") {
+ return message;
+ }
+ }
+
+ if (typeof task.error === "string" && task.error.trim()) {
+ return task.error.trim();
+ }
+
+ const failedCount =
+ task.failed_files ??
+ files.filter((file) => file.status === "failed" || file.status === "error")
+ .length;
+ if (failedCount > 0) {
+ return `${failedCount} file${failedCount !== 1 ? "s" : ""} failed`;
+ }
+ return "Unknown error";
+}
+
export function analyzeTaskFileIngestionFailure(
fileInfo: TaskFileEntry,
taskError?: string,
diff --git a/frontend/tests/core/tasks-unified-panel.spec.ts b/frontend/tests/core/tasks-unified-panel.spec.ts
index 3d6797a29..bf2d2f902 100644
--- a/frontend/tests/core/tasks-unified-panel.spec.ts
+++ b/frontend/tests/core/tasks-unified-panel.spec.ts
@@ -193,8 +193,9 @@ test("completed task with failures keeps failure log in Tasks panel", async ({
page,
"Synthetic ingestion failure for test",
);
+ // Prefer title= on the panel error row — toast description can repeat the same text.
await expect(
- page.getByText("Synthetic ingestion failure for test"),
+ page.getByTitle("Synthetic ingestion failure for test"),
).toBeVisible();
});
@@ -242,7 +243,8 @@ test("completed task with failures requires View click to open tasks panel", asy
await openTasksPanel(page);
await openPastTasksSection(page);
await expandFirstFailureAccordion(page, "Auto-open on partial success");
- await expect(page.getByText("Auto-open on partial success")).toBeVisible();
+ // Prefer title= on the panel error row — toast description can repeat the same text.
+ await expect(page.getByTitle("Auto-open on partial success")).toBeVisible();
});
test("new failed task auto-opens tasks panel", async ({ page }) => {
@@ -287,7 +289,8 @@ test("new failed task auto-opens tasks panel", async ({ page }) => {
await openTasksPanel(page);
await openPastTasksSection(page);
await expandFirstFailureAccordion(page, "Auto-open on failed task");
- await expect(page.getByText("Auto-open on failed task")).toBeVisible();
+ // Prefer title= on the panel error row — toast description can repeat the same text.
+ await expect(page.getByTitle("Auto-open on failed task")).toBeVisible();
});
test("unified panel shows all completed tasks in a single past tasks section", async ({
diff --git a/src/agent.py b/src/agent.py
index a9ee54a0b..bfeec9114 100644
--- a/src/agent.py
+++ b/src/agent.py
@@ -1,3 +1,4 @@
+import uuid
from typing import Any
from services.conversation_persistence_service import conversation_persistence
@@ -9,6 +10,109 @@ logger = get_logger(__name__)
active_conversations: dict[str, dict[str, Any]] = {}
+def _format_provider_error_message(exc: BaseException) -> str:
+ """Return a concise, user-facing provider error string from an exception."""
+ from api.provider_validation import sanitize_provider_error_content
+
+ return sanitize_provider_error_content(str(exc))
+
+
+def _provider_error_display_text(error_text: str, partial_response: str = "") -> str:
+ """Combine any streamed partial answer with the provider error for display/history.
+
+ If the mid-stream text is itself a provider failure dump (often raw JSON), keep
+ only the sanitized error — do not paste the dump above the clean message.
+ """
+ from api.provider_validation import looks_like_provider_error_content
+
+ partial = (partial_response or "").strip()
+ if not partial:
+ return error_text
+ if looks_like_provider_error_content(partial) or "{" in partial:
+ return error_text
+ if error_text in partial:
+ return partial
+ return f"{partial}\n\n{error_text}"
+
+
+def _conversation_thread_in_memory(user_id: str, response_id: str | None) -> bool:
+ """True when response_id is already keyed in the in-process conversation cache."""
+ return bool(
+ response_id
+ and user_id in active_conversations
+ and response_id in active_conversations[user_id]
+ )
+
+
+def _resolve_conversation_store_id(
+ response_id: str | None,
+ thread_id: str | None,
+ *,
+ previous_loaded_from_memory: bool,
+ mint_if_missing: bool = False,
+) -> str | None:
+ """Resolve the OpenRAG conversation id used for persistence.
+
+ Prefer an in-memory thread id so follow-ups stay on one sidebar entry.
+ Never reuse a cold thread id (would overwrite metadata with empty state).
+ Fall back to the stream id, then optionally mint a UUID for failed first turns.
+ """
+ if previous_loaded_from_memory and thread_id:
+ return thread_id
+ if response_id:
+ return response_id
+ if mint_if_missing:
+ return str(uuid.uuid4())
+ return None
+
+
+def _conversation_ended_with_error(user_id: str, response_id: str | None) -> bool:
+ """True when the in-memory thread's last assistant message is a provider error."""
+ if not response_id or user_id not in active_conversations:
+ return False
+ messages = active_conversations[user_id].get(response_id, {}).get("messages") or []
+ for message in reversed(messages):
+ if message.get("role") == "assistant":
+ return bool(message.get("error"))
+ return False
+
+
+async def _persist_error_conversation(
+ user_id: str, store_id: str, conversation_state: dict
+) -> None:
+ """Store a failed turn and claim session ownership.
+
+ Claiming is required: otherwise the client may retry with this response_id and
+ ``_assert_owns`` returns 404 (session_not_found) for an unclaimed session.
+ """
+ from datetime import datetime
+
+ from services.session_ownership_service import session_ownership_service
+
+ conversation_state["last_activity"] = datetime.now()
+ await store_conversation_thread(user_id, store_id, conversation_state)
+ try:
+ await session_ownership_service.claim_session(user_id, store_id)
+ logger.debug(f"Claimed session {store_id} for user {user_id} after stream error")
+ except Exception as e:
+ logger.warning(f"Failed to claim session ownership after stream error: {e}")
+
+
+def _stream_error_chunk(message: str, response_id: str | None = None) -> bytes:
+ """NDJSON error event consumed by the chat streaming client."""
+ import json
+
+ payload: dict[str, Any] = {
+ "status": "failed",
+ "finish_reason": "error",
+ "error": {"message": message},
+ }
+ if response_id:
+ payload["id"] = response_id
+ payload["response_id"] = response_id
+ return (json.dumps(payload) + "\n").encode("utf-8")
+
+
async def get_user_conversations(user_id: str):
"""Get conversation metadata for a user from persistent storage."""
return await conversation_persistence.get_user_conversations(user_id)
@@ -60,6 +164,7 @@ async def store_conversation_thread(user_id: str, response_id: str, conversation
content = first_user_msg.get("content", "")
title = content[:50] + "..." if len(content) > 50 else content
+ user_assistant_messages = [msg for msg in messages if msg.get("role") in ["user", "assistant"]]
metadata_only = {
"response_id": response_id,
"title": title,
@@ -68,11 +173,19 @@ async def store_conversation_thread(user_id: str, response_id: str, conversation
"last_activity": conversation_state.get("last_activity"),
"previous_response_id": conversation_state.get("previous_response_id"),
"filter_id": conversation_state.get("filter_id"),
- "total_messages": len(
- [msg for msg in messages if msg.get("role") in ["user", "assistant"]]
- ),
- # Don't store actual messages - Langflow has them
+ "total_messages": len(user_assistant_messages),
}
+ # Snapshot error-thread turns: Langflow history is incomplete after fresh-session retries.
+ if any(msg.get("error") for msg in user_assistant_messages):
+ metadata_only["messages"] = [
+ {
+ "role": msg.get("role"),
+ "content": msg.get("content"),
+ "timestamp": msg.get("timestamp"),
+ "error": bool(msg.get("error")),
+ }
+ for msg in user_assistant_messages
+ ]
await conversation_persistence.store_conversation_thread(user_id, response_id, metadata_only)
@@ -145,13 +258,16 @@ async def async_response_stream(
if hasattr(chunk, "output_text") and chunk.output_text:
full_response += chunk.output_text
elif hasattr(chunk, "delta") and chunk.delta:
- # Handle delta properly - it might be a dict or string
+ # Handle delta properly - it might be a dict or string.
+ # Do not fall back to str(dict) when content is "" — that pollutes
+ # the transcript with "{'content': ''}" prefixes on error chunks.
if isinstance(chunk.delta, dict):
- delta_text = (
- chunk.delta.get("content", "")
- or chunk.delta.get("text", "")
- or str(chunk.delta)
- )
+ raw_delta = chunk.delta.get("content")
+ if raw_delta is None:
+ raw_delta = chunk.delta.get("text")
+ delta_text = raw_delta if isinstance(raw_delta, str) else ""
+ elif chunk.delta is None:
+ delta_text = ""
else:
delta_text = str(chunk.delta)
full_response += delta_text
@@ -489,6 +605,7 @@ async def async_chat_stream(
previous_response_id: str = None,
filter_id: str = None,
):
+ previous_loaded_from_memory = _conversation_thread_in_memory(user_id, previous_response_id)
# Get the specific conversation thread (or create new one)
conversation_state = get_conversation_thread(user_id, previous_response_id)
@@ -505,52 +622,76 @@ async def async_chat_stream(
full_response = ""
response_id = None
usage_data = None
- async for chunk in async_stream(
- async_client,
- prompt,
- model,
- previous_response_id=previous_response_id,
- log_prefix="agent",
- ):
- # Extract text content to build full response for history
- try:
- import json
+ try:
+ async for chunk in async_stream(
+ async_client,
+ prompt,
+ model,
+ previous_response_id=previous_response_id,
+ log_prefix="agent",
+ ):
+ # Extract text content to build full response for history
+ try:
+ import json
- chunk_data = json.loads(chunk.decode("utf-8"))
- if "delta" in chunk_data and "content" in chunk_data["delta"]:
- full_response += chunk_data["delta"]["content"]
- # Extract response_id from chunk
- if "id" in chunk_data:
- response_id = chunk_data["id"]
- elif "response_id" in chunk_data:
- response_id = chunk_data["response_id"]
- # Capture usage from response.completed event
- if chunk_data.get("type") == "response.completed":
- response_obj = chunk_data.get("response", {})
- usage_data = response_obj.get("usage")
- except Exception:
- pass
- yield chunk
+ chunk_data = json.loads(chunk.decode("utf-8"))
+ if "delta" in chunk_data and "content" in chunk_data["delta"]:
+ full_response += chunk_data["delta"]["content"]
+ # Extract response_id from chunk
+ if "id" in chunk_data:
+ response_id = chunk_data["id"]
+ elif "response_id" in chunk_data:
+ response_id = chunk_data["response_id"]
+ # Capture usage from response.completed event
+ if chunk_data.get("type") == "response.completed":
+ response_obj = chunk_data.get("response", {})
+ usage_data = response_obj.get("usage")
+ except Exception:
+ pass
+ yield chunk
- # Add the complete assistant response to message history with response_id and timestamp
- assistant_message = {
- "role": "assistant",
- "content": full_response,
- "response_id": response_id,
- "timestamp": datetime.now(),
- }
- # Store usage data if available (from response.completed event)
- if usage_data:
- assistant_message["response_data"] = {"usage": usage_data}
- conversation_state["messages"].append(assistant_message)
+ # Add the complete assistant response to message history with response_id and timestamp
+ assistant_message = {
+ "role": "assistant",
+ "content": full_response,
+ "response_id": response_id,
+ "timestamp": datetime.now(),
+ }
+ # Store usage data if available (from response.completed event)
+ if usage_data:
+ assistant_message["response_data"] = {"usage": usage_data}
+ conversation_state["messages"].append(assistant_message)
- # Store the conversation thread with its response_id
- if response_id:
- conversation_state["last_activity"] = datetime.now()
- await store_conversation_thread(user_id, response_id, conversation_state)
- logger.debug(
- f"Stored conversation thread for user {user_id} with response_id: {response_id}"
+ # Store the conversation thread with its response_id
+ if response_id:
+ conversation_state["last_activity"] = datetime.now()
+ await store_conversation_thread(user_id, response_id, conversation_state)
+ logger.debug(
+ f"Stored conversation thread for user {user_id} with response_id: {response_id}"
+ )
+ except Exception as e:
+ logger.error(f"Error in chat stream: {e}", exc_info=True)
+ error_text = _format_provider_error_message(e)
+ display_text = _provider_error_display_text(error_text, full_response)
+ conversation_state["messages"].append(
+ {
+ "role": "assistant",
+ "content": display_text,
+ "timestamp": datetime.now(),
+ "error": True,
+ }
+ )
+ store_id = _resolve_conversation_store_id(
+ response_id,
+ previous_response_id,
+ previous_loaded_from_memory=previous_loaded_from_memory,
+ mint_if_missing=True,
)
+ try:
+ await _persist_error_conversation(user_id, store_id, conversation_state)
+ except Exception as store_error:
+ logger.error(f"Failed to store error conversation: {store_error}")
+ yield _stream_error_chunk(display_text, store_id)
# Async langflow function with conversation storage (non-streaming)
@@ -561,6 +702,7 @@ async def async_langflow_chat(
user_id: str,
extra_headers: dict = None,
previous_response_id: str = None,
+ conversation_id: str = None,
store_conversation: bool = True,
filter_id: str = None,
):
@@ -568,11 +710,13 @@ async def async_langflow_chat(
"async_langflow_chat called",
user_id=user_id,
previous_response_id=previous_response_id,
+ conversation_id=conversation_id,
)
+ thread_id = conversation_id or previous_response_id
if store_conversation:
# Get the specific conversation thread (or create new one)
- conversation_state = get_conversation_thread(user_id, previous_response_id)
+ conversation_state = get_conversation_thread(user_id, thread_id)
logger.debug(
"Got langflow conversation state",
message_count=len(conversation_state["messages"]),
@@ -730,23 +874,11 @@ async def async_langflow_chat(
except Exception as e:
logger.warning(f"Failed to claim session ownership: {e}")
- logger.debug(
- f"Stored langflow conversation thread for user {user_id} with response_id: {response_id}"
- )
logger.debug(
"Stored langflow conversation thread",
user_id=user_id,
response_id=response_id,
)
-
- # Debug: Check what's in user_conversations now
- conversations = await get_user_conversations(user_id)
- logger.debug(
- "User conversations updated",
- user_id=user_id,
- conversation_count=len(conversations),
- conversation_ids=list(conversations.keys()),
- )
else:
logger.warning("No response_id received from langflow, conversation not stored")
@@ -761,27 +893,35 @@ async def async_langflow_chat_stream(
user_id: str,
extra_headers: dict = None,
previous_response_id: str = None,
+ conversation_id: str = None,
filter_id: str = None,
):
logger.debug(
"async_langflow_chat_stream called",
user_id=user_id,
previous_response_id=previous_response_id,
+ conversation_id=conversation_id,
)
- # Get the specific conversation thread (or create new one)
- conversation_state = get_conversation_thread(user_id, previous_response_id)
+ # conversation_id keeps the OpenRAG sidebar thread; previous_response_id chains
+ # the Langflow session. After errors the client omits previous_response_id.
+ thread_id = conversation_id or previous_response_id
+ previous_loaded_from_memory = _conversation_thread_in_memory(user_id, thread_id)
+ conversation_state = get_conversation_thread(user_id, thread_id)
- # Add user message to conversation with timestamp
from datetime import datetime
- user_message = {"role": "user", "content": prompt, "timestamp": datetime.now()}
- conversation_state["messages"].append(user_message)
-
- # Store filter_id in conversation state if provided
+ conversation_state["messages"].append(
+ {"role": "user", "content": prompt, "timestamp": datetime.now()}
+ )
if filter_id:
conversation_state["filter_id"] = filter_id
+ # Safety net: never continue a Langflow session that already ended in error.
+ langflow_previous_response_id = previous_response_id
+ if previous_response_id and _conversation_ended_with_error(user_id, thread_id):
+ langflow_previous_response_id = None
+
full_response = ""
response_id = None
usage_data = None
@@ -794,7 +934,7 @@ async def async_langflow_chat_stream(
prompt,
flow_id,
extra_headers=extra_headers,
- previous_response_id=previous_response_id,
+ previous_response_id=langflow_previous_response_id,
log_prefix="langflow",
):
# Extract text content to build full response for history
@@ -812,13 +952,32 @@ async def async_langflow_chat_stream(
elif "response_id" in chunk_data:
response_id = chunk_data["response_id"]
- # Check for error status
+ # Check for error status. Do not forward bare Langflow error
+ # chunks — they often lack a useful message and the client would
+ # show a generic failure before our sanitized terminal chunk.
if (
chunk_data.get("finish_reason") == "error"
or chunk_data.get("status") == "failed"
):
error_occurred = True
logger.error("Error detected in Langflow stream chunk")
+ err = chunk_data.get("error")
+ if isinstance(err, dict):
+ err_msg = err.get("message")
+ else:
+ err_msg = err
+ if isinstance(err_msg, str) and err_msg.strip():
+ if err_msg not in full_response:
+ full_response = (
+ f"{full_response}\n\n{err_msg}" if full_response else err_msg
+ )
+ elif (
+ isinstance(chunk_data.get("message"), str) and chunk_data["message"].strip()
+ ):
+ msg = chunk_data["message"]
+ if msg not in full_response:
+ full_response = f"{full_response}\n\n{msg}" if full_response else msg
+ continue
# Capture usage from response.completed event
if chunk_data.get("type") == "response.completed":
response_obj = chunk_data.get("response", {})
@@ -827,7 +986,39 @@ async def async_langflow_chat_stream(
pass
yield chunk
- # Add the complete assistant response to message history with response_id, timestamp, and function call data
+ from api.provider_validation import (
+ looks_like_provider_error_content,
+ resolve_chat_stream_error_message,
+ )
+
+ # Credential failures are often streamed as plain assistant text with no
+ # exception. Sanitize, mark as error, and emit a terminal error chunk so
+ # the client shows the error card instead of raw JSON.
+ if error_occurred or looks_like_provider_error_content(full_response):
+ display_text = resolve_chat_stream_error_message(full_response)
+ conversation_state["messages"].append(
+ {
+ "role": "assistant",
+ "content": display_text,
+ "response_id": response_id,
+ "timestamp": datetime.now(),
+ "chunks": collected_chunks,
+ "error": True,
+ }
+ )
+ store_id = _resolve_conversation_store_id(
+ response_id,
+ thread_id,
+ previous_loaded_from_memory=previous_loaded_from_memory,
+ mint_if_missing=True,
+ )
+ try:
+ await _persist_error_conversation(user_id, store_id, conversation_state)
+ except Exception as store_error:
+ logger.error(f"Failed to store error conversation: {store_error}")
+ yield _stream_error_chunk(display_text, store_id)
+ return
+
if full_response:
assistant_message = {
"role": "assistant",
@@ -841,50 +1032,48 @@ async def async_langflow_chat_stream(
assistant_message["response_data"] = {"usage": usage_data}
conversation_state["messages"].append(assistant_message)
- # Store the conversation thread with its response_id
- if response_id:
+ persist_id = _resolve_conversation_store_id(
+ response_id,
+ thread_id,
+ previous_loaded_from_memory=previous_loaded_from_memory,
+ )
+ if persist_id:
conversation_state["last_activity"] = datetime.now()
- await store_conversation_thread(user_id, response_id, conversation_state)
-
+ await store_conversation_thread(user_id, persist_id, conversation_state)
try:
from services.session_ownership_service import session_ownership_service
- await session_ownership_service.claim_session(user_id, response_id)
- logger.debug(f"Claimed session {response_id} for user {user_id}")
+ await session_ownership_service.claim_session(user_id, persist_id)
except Exception as e:
logger.warning(f"Failed to claim session ownership: {e}")
-
- logger.debug(
- f"Stored langflow conversation thread for user {user_id} with response_id: {response_id}"
- )
except Exception as e:
- # Log the error
logger.error(f"Error in langflow chat stream: {e}", exc_info=True)
- error_occurred = True
+ from api.provider_validation import resolve_chat_stream_error_message
- # Store error message in conversation history so it persists
- error_message = {
- "role": "assistant",
- "content": f"Sorry, I encountered an error: {str(e)}",
- "timestamp": datetime.now(),
- "error": True,
- }
- conversation_state["messages"].append(error_message)
-
- # Try to store the conversation with error message
- # Use a temporary response_id if we don't have one
- if not response_id:
- response_id = f"error_{user_id}_{int(datetime.now().timestamp())}"
+ error_text = resolve_chat_stream_error_message(e)
+ display_text = _provider_error_display_text(error_text, full_response)
+ if full_response:
+ display_text = resolve_chat_stream_error_message(display_text)
+ conversation_state["messages"].append(
+ {
+ "role": "assistant",
+ "content": display_text,
+ "timestamp": datetime.now(),
+ "error": True,
+ }
+ )
+ store_id = _resolve_conversation_store_id(
+ response_id,
+ thread_id,
+ previous_loaded_from_memory=previous_loaded_from_memory,
+ mint_if_missing=True,
+ )
try:
- conversation_state["last_activity"] = datetime.now()
- await store_conversation_thread(user_id, response_id, conversation_state)
- logger.debug(f"Stored conversation with error for user {user_id}")
+ await _persist_error_conversation(user_id, store_id, conversation_state)
except Exception as store_error:
logger.error(f"Failed to store error conversation: {store_error}")
-
- # Re-raise the exception so it propagates to the API layer
- raise
+ yield _stream_error_chunk(display_text, store_id)
async def delete_user_conversation(user_id: str, response_id: str) -> bool:
diff --git a/src/api/chat.py b/src/api/chat.py
index 23e8b5eb3..a3baa7e1c 100644
--- a/src/api/chat.py
+++ b/src/api/chat.py
@@ -1,17 +1,16 @@
-from typing import Optional, Any, Dict
+from typing import Any
from fastapi import Depends, HTTPException
-from pydantic import BaseModel
from fastapi.responses import JSONResponse, StreamingResponse
-from utils.logging_config import get_logger
+from pydantic import BaseModel
from dependencies import (
get_chat_service,
get_session_manager,
- get_current_user,
require_permission,
)
from session_manager import User
+from utils.logging_config import get_logger
logger = get_logger(__name__)
@@ -20,7 +19,7 @@ def _openrag_user_id(user: User) -> str:
return getattr(user, "db_user_id", None) or user.user_id
-async def _assert_owns(session_id: Optional[str], user_id: str) -> None:
+async def _assert_owns(session_id: str | None, user_id: str) -> None:
"""Raise 403 if `session_id` is set but not owned by `user_id`.
No-op when `session_id` is None (new conversation, nothing to check).
@@ -30,6 +29,7 @@ async def _assert_owns(session_id: Optional[str], user_id: str) -> None:
if not session_id:
return
from services.session_ownership_service import session_ownership_service
+
owner = await session_ownership_service.get_session_owner(session_id)
if owner is None:
raise HTTPException(status_code=404, detail={"error": "session_not_found"})
@@ -39,12 +39,15 @@ async def _assert_owns(session_id: Optional[str], user_id: str) -> None:
class ChatBody(BaseModel):
prompt: str
- previous_response_id: Optional[str] = None
+ previous_response_id: str | None = None
+ # OpenRAG sidebar/thread id. Distinct from previous_response_id so a retry
+ # after an error can stay in the same chat while starting a fresh Langflow session.
+ conversation_id: str | None = None
stream: bool = False
- filters: Optional[Dict[str, Any]] = None
+ filters: dict[str, Any] | None = None
limit: int = 10
scoreThreshold: float = 0
- filter_id: Optional[str] = None
+ filter_id: str | None = None
async def chat_endpoint(
@@ -64,9 +67,11 @@ async def chat_endpoint(
if body.filters:
from auth_context import set_search_filters
+
set_search_filters(body.filters)
- from auth_context import set_search_limit, set_score_threshold
+ from auth_context import set_score_threshold, set_search_limit
+
set_search_limit(body.limit)
set_score_threshold(body.scoreThreshold)
@@ -114,14 +119,17 @@ async def langflow_endpoint(
storage_user_id = _openrag_user_id(user)
await _assert_owns(body.previous_response_id, storage_user_id)
+ await _assert_owns(body.conversation_id, storage_user_id)
jwt_token = user.jwt_token
if body.filters:
from auth_context import set_search_filters
+
set_search_filters(body.filters)
- from auth_context import set_search_limit, set_score_threshold
+ from auth_context import set_score_threshold, set_search_limit
+
set_search_limit(body.limit)
set_score_threshold(body.scoreThreshold)
@@ -133,6 +141,7 @@ async def langflow_endpoint(
user.user_id,
jwt_token,
previous_response_id=body.previous_response_id,
+ conversation_id=body.conversation_id,
stream=True,
filter_id=body.filter_id,
owner=user.user_id,
@@ -154,6 +163,7 @@ async def langflow_endpoint(
user.user_id,
jwt_token,
previous_response_id=body.previous_response_id,
+ conversation_id=body.conversation_id,
stream=False,
filter_id=body.filter_id,
owner=user.user_id,
@@ -163,11 +173,9 @@ async def langflow_endpoint(
)
return JSONResponse(result)
- except Exception as e:
+ except Exception:
logger.exception("[CHAT] Langflow request failed")
- return JSONResponse(
- {"error": f"Langflow request failed: {str(e)}"}, status_code=500
- )
+ return JSONResponse({"error": "Langflow request failed"}, status_code=500)
async def chat_history_endpoint(
@@ -178,11 +186,9 @@ async def chat_history_endpoint(
try:
history = await chat_service.get_chat_history(_openrag_user_id(user))
return JSONResponse(history)
- except Exception as e:
+ except Exception:
logger.exception("[CHAT] Failed to get chat history")
- return JSONResponse(
- {"error": f"Failed to get chat history: {str(e)}"}, status_code=500
- )
+ return JSONResponse({"error": "Failed to get chat history"}, status_code=500)
async def langflow_history_endpoint(
@@ -193,11 +199,9 @@ async def langflow_history_endpoint(
try:
history = await chat_service.get_langflow_history(_openrag_user_id(user))
return JSONResponse(history)
- except Exception as e:
+ except Exception:
logger.exception("[CHAT] Failed to get langflow history")
- return JSONResponse(
- {"error": f"Failed to get langflow history: {str(e)}"}, status_code=500
- )
+ return JSONResponse({"error": "Failed to get langflow history"}, status_code=500)
async def delete_session_endpoint(
@@ -215,11 +219,8 @@ async def delete_session_endpoint(
return JSONResponse({"message": "Session deleted successfully"})
else:
return JSONResponse(
- {"error": result.get("error", "Failed to delete session")},
- status_code=500
+ {"error": result.get("error", "Failed to delete session")}, status_code=500
)
- except Exception as e:
- logger.error(f"Error deleting session: {e}")
- return JSONResponse(
- {"error": f"Failed to delete session: {str(e)}"}, status_code=500
- )
+ except Exception:
+ logger.exception("Error deleting session")
+ return JSONResponse({"error": "Failed to delete session"}, status_code=500)
diff --git a/src/api/provider_health.py b/src/api/provider_health.py
index adb27636a..bc5aab9a7 100644
--- a/src/api/provider_health.py
+++ b/src/api/provider_health.py
@@ -6,7 +6,7 @@ import httpx
from fastapi import Depends
from fastapi.responses import JSONResponse
-from api.provider_validation import validate_provider_setup
+from api.provider_validation import format_provider_error_message, validate_provider_setup
from config.settings import get_openrag_config
from dependencies import require_permission
from session_manager import User
@@ -195,10 +195,10 @@ async def check_provider_health(
llm_error = None # Don't treat as error
logger.info(f"LLM provider ({provider}) appears busy: {str(e)}")
else:
- llm_error = str(e)
+ llm_error = format_provider_error_message(e)
logger.error(f"LLM provider ({provider}) validation timed out: {llm_error}")
except Exception as e:
- llm_error = str(e)
+ llm_error = format_provider_error_message(e)
logger.error(f"LLM provider ({provider}) validation failed: {llm_error}")
# Validate embedding provider
@@ -229,12 +229,12 @@ async def check_provider_health(
embedding_error = None # Don't treat as error
logger.info(f"Embedding provider ({embedding_provider}) appears busy: {str(e)}")
else:
- embedding_error = str(e)
+ embedding_error = format_provider_error_message(e)
logger.error(
f"Embedding provider ({embedding_provider}) validation timed out: {embedding_error}"
)
except Exception as e:
- embedding_error = str(e)
+ embedding_error = format_provider_error_message(e)
logger.error(
f"Embedding provider ({embedding_provider}) validation failed: {embedding_error}"
)
@@ -283,7 +283,7 @@ async def check_provider_health(
except Exception as e:
if _health_leader_key:
provider_health_cache.release_error(_health_leader_key)
- error_message = str(e)
+ error_message = format_provider_error_message(e)
logger.error(f"Provider health check failed for {provider}: {error_message}")
return JSONResponse(
diff --git a/src/api/provider_validation.py b/src/api/provider_validation.py
index 431663210..494a80098 100644
--- a/src/api/provider_validation.py
+++ b/src/api/provider_validation.py
@@ -1,17 +1,336 @@
"""Provider validation utilities for testing API keys and models during onboarding."""
import json
+import re
+
import httpx
+
from utils.container_utils import transform_localhost_url
from utils.logging_config import get_logger
logger = get_logger(__name__)
+
+# Peel leading "Error <label>:" wrappers (e.g. Langflow graph/component noise).
+# Requires a label after "Error" so bare "Error: …" messages are preserved.
+_ERROR_LABEL_PREFIX_RE = re.compile(r"^Error\s+\S[^:]*:\s*", re.IGNORECASE)
+
+
+def _strip_error_label_prefixes(text: str) -> str:
+ """Remove leading ``Error <label>:`` segments; keep the innermost message."""
+ cleaned = text.strip()
+ while True:
+ updated = _ERROR_LABEL_PREFIX_RE.sub("", cleaned, count=1).strip()
+ if updated == cleaned:
+ return cleaned
+ cleaned = updated
+
+
+_PROVIDER_CREDENTIAL_ERROR_MARKERS = (
+ "incorrect api key",
+ "invalid api key",
+ "invalid_api_key",
+ "api key could not be found",
+ "api key is invalid",
+ "api key has been revoked",
+ "api key revoked",
+ "revoked api key",
+ "provided api key could not be found",
+ # IBM IAM returns this when the key still exists but is toggled off.
+ "api key is disabled",
+ "api key disabled",
+ "authentication_error",
+ "failed to authenticate",
+ "invalid x-api-key",
+ "unauthorized",
+ "authentication failed",
+ "invalid credentials",
+ "could not authenticate",
+)
+
+
+_PROVIDER_ERROR_CONTENT_MARKERS = _PROVIDER_CREDENTIAL_ERROR_MARKERS + (
+ "rate limit",
+ "rate_limit",
+ "permission denied",
+ "quota exceeded",
+ "provider request failed",
+ "insufficient_quota",
+)
+
+
+def _extract_balanced_json_object(text: str) -> str | None:
+ """Return the first top-level `{...}` substring, ignoring trailing junk."""
+ start = text.find("{")
+ if start < 0:
+ return None
+ depth = 0
+ in_string = False
+ escape = False
+ for i in range(start, len(text)):
+ ch = text[i]
+ if in_string:
+ if escape:
+ escape = False
+ elif ch == "\\":
+ escape = True
+ elif ch == '"':
+ in_string = False
+ continue
+ if ch == '"':
+ in_string = True
+ elif ch == "{":
+ depth += 1
+ elif ch == "}":
+ depth -= 1
+ if depth == 0:
+ return text[start : i + 1]
+ return None
+
+
+def format_provider_error_message(exc: BaseException | str) -> str:
+ """Return a concise, user-facing provider error string from an exception or text."""
+ text = _strip_error_label_prefixes((str(exc) if not isinstance(exc, str) else exc).strip())
+ if not text:
+ return "An error occurred while generating a response."
+
+ parsed = _parse_json_error_message(text)
+ if parsed != text:
+ # Pure JSON (or JSON-shaped) payloads: prefer the extracted message only.
+ return _strip_error_label_prefixes(parsed)
+
+ json_blob = _extract_balanced_json_object(text)
+ if json_blob:
+ nested = _parse_json_error_message(json_blob)
+ if nested != json_blob:
+ # Embedded provider JSON already has the real message — drop wrappers.
+ return _strip_error_label_prefixes(nested)
+
+ return text
+
+
+def is_provider_credential_error(text: str | BaseException | None) -> bool:
+ """True when the error indicates an invalid, missing, or revoked provider API key."""
+ if text is None:
+ return False
+ lowered = (str(text) if not isinstance(text, str) else text).lower()
+ return any(marker in lowered for marker in _PROVIDER_CREDENTIAL_ERROR_MARKERS)
+
+
+_GENERIC_UPSTREAM_ERROR_MARKERS = (
+ "an unknown error occurred",
+ "an error occurred while generating a response",
+ "unknown error occurred",
+ "something went wrong",
+)
+
+
+def is_generic_upstream_error(text: str | BaseException | None) -> bool:
+ """True when upstream returned an opaque error with no actionable detail."""
+ if text is None:
+ return True
+ lowered = (str(text) if not isinstance(text, str) else text).strip().lower().rstrip(".")
+ if not lowered:
+ return True
+ generics = {marker.rstrip(".") for marker in _GENERIC_UPSTREAM_ERROR_MARKERS}
+ return lowered in generics
+
+
+def looks_like_provider_error_content(text: str | None) -> bool:
+ """True when assistant text looks like a provider/auth failure rather than a normal reply."""
+ if not text or not text.strip():
+ return False
+ content = text.strip()
+ if content.startswith("Error:"):
+ return True
+ if is_generic_upstream_error(content):
+ return True
+ lowered = content.lower()
+ if any(marker in lowered for marker in _PROVIDER_ERROR_CONTENT_MARKERS):
+ return True
+ # Raw provider payloads often embed JSON error objects.
+ if "{" in content and (
+ '"error"' in lowered
+ or '"errormessage"' in lowered
+ or '"errorcode"' in lowered
+ or '"errors"' in lowered
+ ):
+ return True
+ return False
+
+
+def sanitize_provider_error_content(text: str | BaseException | None) -> str:
+ """Format provider error text and fall back when JSON still leaks through."""
+ if text is None:
+ return "An error occurred while generating a response."
+ if not isinstance(text, str):
+ text = str(text)
+ if not text.strip():
+ return "An error occurred while generating a response."
+ cleaned = format_provider_error_message(text)
+ if "{" in cleaned or "}" in cleaned:
+ if is_provider_credential_error(text) or is_provider_credential_error(cleaned):
+ return (
+ "The configured API key is invalid or has been revoked. "
+ "Update it in Settings and retry."
+ )
+ # Strip the first JSON object as a last resort.
+ json_start = cleaned.find("{")
+ prefix = cleaned[:json_start].rstrip(": ").strip()
+ return prefix or "An error occurred while generating a response."
+ return cleaned
+
+
+_LANGFLOW_TRANSPORT_ERROR_MARKERS = (
+ "server disconnected",
+ "disconnected without sending a response",
+ "remote protocol error",
+ "connection reset",
+ "broken pipe",
+ "all connection attempts failed",
+ "connection refused",
+)
+
+
+def is_langflow_transport_error(text: str | BaseException | None) -> bool:
+ """True when Langflow dropped the HTTP connection mid-ingest."""
+ if text is None:
+ return False
+ lowered = (str(text) if not isinstance(text, str) else text).lower()
+ return any(marker in lowered for marker in _LANGFLOW_TRANSPORT_ERROR_MARKERS)
+
+
+async def _probe_provider_credential_error(
+ *,
+ provider: str | None,
+ api_key: str | None,
+ endpoint: str | None = None,
+ project_id: str | None = None,
+ embedding_model: str | None = None,
+ llm_model: str | None = None,
+) -> str | None:
+ """Run a lightweight provider check; return a cleaned credential error if auth fails."""
+ if not provider:
+ return None
+ try:
+ await validate_provider_setup(
+ provider=provider,
+ api_key=api_key,
+ embedding_model=embedding_model,
+ llm_model=llm_model,
+ endpoint=endpoint,
+ project_id=project_id,
+ test_completion=False,
+ )
+ except Exception as probe_exc:
+ cleaned = sanitize_provider_error_content(probe_exc)
+ if is_provider_credential_error(probe_exc) or is_provider_credential_error(cleaned):
+ return cleaned
+ return None
+
+
+async def probe_provider_credential_error() -> str | None:
+ """Return a credential error if any ingest-relevant provider key fails auth.
+
+ Langflow receives every configured provider key, so a revoked watsonx key can
+ crash ingest even when the selected embedding provider is OpenAI.
+ """
+ from config.settings import get_openrag_config
+
+ config = get_openrag_config()
+ checked: set[str] = set()
+
+ candidates: list[tuple[str | None, object, str | None, str | None]] = [
+ (
+ config.knowledge.embedding_provider,
+ config.get_embedding_provider_config(),
+ config.knowledge.embedding_model,
+ None,
+ ),
+ (
+ config.agent.llm_provider,
+ config.get_llm_provider_config(),
+ None,
+ config.agent.llm_model,
+ ),
+ ]
+ for name in ("openai", "anthropic", "watsonx", "ollama"):
+ candidates.append((name, getattr(config.providers, name, None), None, None))
+
+ for provider, provider_config, embedding_model, llm_model in candidates:
+ if not provider or provider in checked or provider_config is None:
+ continue
+ checked.add(provider)
+ api_key = getattr(provider_config, "api_key", None)
+ endpoint = getattr(provider_config, "endpoint", None)
+ if provider == "ollama":
+ if not endpoint:
+ continue
+ elif (
+ provider
+ not in (
+ config.knowledge.embedding_provider,
+ config.agent.llm_provider,
+ )
+ and not api_key
+ ):
+ continue
+
+ error = await _probe_provider_credential_error(
+ provider=provider,
+ api_key=api_key,
+ endpoint=endpoint,
+ project_id=getattr(provider_config, "project_id", None),
+ embedding_model=embedding_model,
+ llm_model=llm_model,
+ )
+ if error:
+ return error
+ return None
+
+
+async def resolve_ingest_error_message(exc: BaseException | str) -> str:
+ """Prefer a credential message when Langflow fails auth or only reports a disconnect."""
+ raw = (str(exc) if not isinstance(exc, str) else exc).strip() or "Ingestion failed"
+
+ cleaned = sanitize_provider_error_content(raw)
+ if is_provider_credential_error(raw) or is_provider_credential_error(cleaned):
+ return cleaned
+
+ if is_langflow_transport_error(raw):
+ probe = await probe_provider_credential_error()
+ if probe:
+ logger.info(
+ "Ingest transport failure attributed to provider credentials",
+ transport_error=raw,
+ credential_error=probe,
+ )
+ return probe
+
+ # Prefer the sanitized form whenever JSON / boilerplate was stripped.
+ if cleaned != raw and "{" not in cleaned:
+ return cleaned
+ return raw
+
+
+def resolve_chat_stream_error_message(text: str | BaseException | None) -> str:
+ """Sanitize chat stream errors without probing providers.
+
+ Do not infer API-key failures from opaque upstream text — that can hide the
+ real failure. Callers should break Langflow session chaining after errors so
+ the next turn is a fresh session that returns the detailed upstream message.
+ """
+ raw = (str(text) if not isinstance(text, str) else text) if text is not None else ""
+ if not raw.strip():
+ return "An error occurred while generating a response."
+ return sanitize_provider_error_content(raw)
+
+
def _parse_json_error_message(error_text: str) -> str:
"""Parse JSON error message and extract just the message field."""
try:
# Try to parse as JSON
error_data = json.loads(error_text)
-
+
if isinstance(error_data, dict):
# WatsonX format: {"errors": [{"code": "...", "message": "..."}], ...}
if "errors" in error_data and isinstance(error_data["errors"], list):
@@ -23,7 +342,7 @@ def _parse_json_error_message(error_text: str) -> str:
code = errors[0].get("code", "")
if code:
return f"Error: {code}"
-
+
# OpenAI format: {"error": {"message": "...", "type": "...", "code": "..."}}
if "error" in error_data:
error_obj = error_data["error"]
@@ -31,17 +350,21 @@ def _parse_json_error_message(error_text: str) -> str:
message = error_obj.get("message", "")
if message:
return message
-
+
# Direct message field
if "message" in error_data:
return error_data["message"]
-
+
+ # IBM IAM format: {"errorCode": "...", "errorMessage": "...", "context": {...}}
+ if "errorMessage" in error_data:
+ return error_data["errorMessage"]
+
# Generic format: {"detail": "..."}
if "detail" in error_data:
return error_data["detail"]
except (json.JSONDecodeError, ValueError, TypeError):
pass
-
+
# Return original text if not JSON or can't parse
return error_text
@@ -51,7 +374,7 @@ def _extract_error_details(response: httpx.Response) -> str:
try:
# Try to parse JSON error response
error_data = response.json()
-
+
# Common error response formats
if isinstance(error_data, dict):
# WatsonX format: {"errors": [{"code": "...", "message": "..."}], ...}
@@ -66,7 +389,7 @@ def _extract_error_details(response: httpx.Response) -> str:
code = errors[0].get("code", "")
if code:
return f"Error: {code}"
-
+
# OpenAI format: {"error": {"message": "...", "type": "...", "code": "..."}}
if "error" in error_data:
error_obj = error_data["error"]
@@ -81,22 +404,26 @@ def _extract_error_details(response: httpx.Response) -> str:
if code:
details += f" (code: {code})"
return details
-
- # Anthropic format: {"error": {"message": "...", "type": "..."}}
+
+ # Anthropic / generic: {"message": "..."}
if "message" in error_data:
return error_data["message"]
-
- # Generic format: {"message": "..."}
+
+ # IBM IAM format: {"errorCode": "...", "errorMessage": "...", "context": {...}}
+ if "errorMessage" in error_data:
+ return error_data["errorMessage"]
+
+ # Generic format: {"detail": "..."}
if "detail" in error_data:
return error_data["detail"]
-
+
# If JSON parsing worked but no structured error found, try parsing text
response_text = response.text[:500]
parsed = _parse_json_error_message(response_text)
if parsed != response_text:
return parsed
return response_text
-
+
except (json.JSONDecodeError, ValueError):
# If JSON parsing fails, try parsing the text as JSON string
response_text = response.text[:500] if response.text else f"HTTP {response.status_code}"
@@ -134,7 +461,9 @@ async def validate_provider_setup(
provider_lower = provider.lower()
try:
- logger.info(f"Starting validation for provider: {provider_lower} (test_completion={test_completion})")
+ logger.info(
+ f"Starting validation for provider: {provider_lower} (test_completion={test_completion})"
+ )
if test_completion:
# Full validation with completion/embedding tests (consumes credits)
@@ -236,7 +565,7 @@ async def test_embedding(
# OpenAI validation functions
async def _test_openai_lightweight_health(api_key: str) -> None:
"""Test OpenAI API key validity with lightweight check.
-
+
Only checks if the API key is valid without consuming credits.
Uses the /v1/models endpoint which doesn't consume credits.
"""
@@ -256,14 +585,16 @@ async def _test_openai_lightweight_health(api_key: str) -> None:
if response.status_code != 200:
error_details = _extract_error_details(response)
- logger.error(f"OpenAI lightweight health check failed: {response.status_code} - {error_details}")
+ logger.error(
+ f"OpenAI lightweight health check failed: {response.status_code} - {error_details}"
+ )
raise Exception(f"OpenAI API key validation failed: {error_details}")
logger.info("OpenAI lightweight health check passed")
except httpx.TimeoutException:
logger.error("OpenAI lightweight health check timed out")
- raise Exception("OpenAI API request timed out")
+ raise Exception("OpenAI API request timed out") from None
except Exception as e:
logger.error(f"OpenAI lightweight health check failed: {str(e)}")
raise
@@ -280,9 +611,7 @@ async def _test_openai_completion_with_tools(api_key: str, llm_model: str) -> No
# Simple tool calling test
base_payload = {
"model": llm_model,
- "messages": [
- {"role": "user", "content": "What tools do you have available?"}
- ],
+ "messages": [{"role": "user", "content": "What tools do you have available?"}],
"tools": [
{
"type": "function",
@@ -292,14 +621,11 @@ async def _test_openai_completion_with_tools(api_key: str, llm_model: str) -> No
"parameters": {
"type": "object",
"properties": {
- "location": {
- "type": "string",
- "description": "The city and state"
- }
+ "location": {"type": "string", "description": "The city and state"}
},
- "required": ["location"]
- }
- }
+ "required": ["location"],
+ },
+ },
}
],
}
@@ -316,7 +642,9 @@ async def _test_openai_completion_with_tools(api_key: str, llm_model: str) -> No
# If max_tokens doesn't work, try with max_completion_tokens
if response.status_code != 200:
- logger.warning("[API] max_tokens parameter failed, trying max_completion_tokens instead")
+ logger.warning(
+ "[API] max_tokens parameter failed, trying max_completion_tokens instead"
+ )
payload = {**base_payload, "max_completion_tokens": 50}
response = await client.post(
"https://api.openai.com/v1/chat/completions",
@@ -327,14 +655,16 @@ async def _test_openai_completion_with_tools(api_key: str, llm_model: str) -> No
if response.status_code != 200:
error_details = _extract_error_details(response)
- logger.error(f"OpenAI completion test failed: {response.status_code} - {error_details}")
+ logger.error(
+ f"OpenAI completion test failed: {response.status_code} - {error_details}"
+ )
raise Exception(f"OpenAI API error: {error_details}")
logger.info("OpenAI completion with tool calling test passed")
except httpx.TimeoutException:
logger.error("OpenAI completion test timed out")
- raise Exception("Request timed out")
+ raise Exception("Request timed out") from None
except Exception as e:
logger.error(f"OpenAI completion test failed: {str(e)}")
raise
@@ -363,7 +693,9 @@ async def _test_openai_embedding(api_key: str, embedding_model: str) -> None:
if response.status_code != 200:
error_details = _extract_error_details(response)
- logger.error(f"OpenAI embedding test failed: {response.status_code} - {error_details}")
+ logger.error(
+ f"OpenAI embedding test failed: {response.status_code} - {error_details}"
+ )
raise Exception(f"OpenAI API error: {error_details}")
data = response.json()
@@ -374,18 +706,16 @@ async def _test_openai_embedding(api_key: str, embedding_model: str) -> None:
except httpx.TimeoutException:
logger.error("OpenAI embedding test timed out")
- raise Exception("Request timed out")
+ raise Exception("Request timed out") from None
except Exception as e:
logger.error(f"OpenAI embedding test failed: {str(e)}")
raise
# IBM Watson validation functions
-async def _test_watsonx_lightweight_health(
- api_key: str, endpoint: str, project_id: str
-) -> None:
+async def _test_watsonx_lightweight_health(api_key: str, endpoint: str, project_id: str) -> None:
"""Test WatsonX API key validity with lightweight check.
-
+
Only checks if the API key is valid by getting a bearer token.
Does not consume credits by avoiding model inference requests.
"""
@@ -404,7 +734,9 @@ async def _test_watsonx_lightweight_health(
if token_response.status_code != 200:
error_details = _extract_error_details(token_response)
- logger.error(f"IBM IAM token request failed: {token_response.status_code} - {error_details}")
+ logger.error(
+ f"IBM IAM token request failed: {token_response.status_code} - {error_details}"
+ )
raise Exception(f"Failed to authenticate with IBM Watson: {error_details}")
bearer_token = token_response.json().get("access_token")
@@ -415,7 +747,7 @@ async def _test_watsonx_lightweight_health(
except httpx.TimeoutException:
logger.error("WatsonX lightweight health check timed out")
- raise Exception("WatsonX API request timed out")
+ raise Exception("WatsonX API request timed out") from None
except Exception as e:
logger.error(f"WatsonX lightweight health check failed: {str(e)}")
raise
@@ -440,7 +772,9 @@ async def _test_watsonx_completion_with_tools(
if token_response.status_code != 200:
error_details = _extract_error_details(token_response)
- logger.error(f"IBM IAM token request failed: {token_response.status_code} - {error_details}")
+ logger.error(
+ f"IBM IAM token request failed: {token_response.status_code} - {error_details}"
+ )
raise Exception(f"Failed to authenticate with IBM Watson: {error_details}")
bearer_token = token_response.json().get("access_token")
@@ -458,9 +792,7 @@ async def _test_watsonx_completion_with_tools(
payload = {
"model_id": llm_model,
"project_id": project_id,
- "messages": [
- {"role": "user", "content": "What tools do you have available?"}
- ],
+ "messages": [{"role": "user", "content": "What tools do you have available?"}],
"tools": [
{
"type": "function",
@@ -470,14 +802,11 @@ async def _test_watsonx_completion_with_tools(
"parameters": {
"type": "object",
"properties": {
- "location": {
- "type": "string",
- "description": "The city and state"
- }
+ "location": {"type": "string", "description": "The city and state"}
},
- "required": ["location"]
- }
- }
+ "required": ["location"],
+ },
+ },
}
],
"max_tokens": 50,
@@ -494,7 +823,9 @@ async def _test_watsonx_completion_with_tools(
if response.status_code != 200:
error_details = _extract_error_details(response)
- logger.error(f"IBM Watson completion test failed: {response.status_code} - {error_details}")
+ logger.error(
+ f"IBM Watson completion test failed: {response.status_code} - {error_details}"
+ )
# If error_details is still JSON, parse it to extract just the message
parsed_details = _parse_json_error_message(error_details)
raise Exception(f"IBM Watson API error: {parsed_details}")
@@ -503,7 +834,7 @@ async def _test_watsonx_completion_with_tools(
except httpx.TimeoutException:
logger.error("IBM Watson completion test timed out")
- raise Exception("Request timed out")
+ raise Exception("Request timed out") from None
except Exception as e:
logger.error(f"IBM Watson completion test failed: {str(e)}")
# If the error message contains JSON, parse it to extract just the message
@@ -512,7 +843,7 @@ async def _test_watsonx_completion_with_tools(
json_part = error_str.split("IBM Watson API error: ", 1)[1]
parsed_message = _parse_json_error_message(json_part)
if parsed_message != json_part:
- raise Exception(f"IBM Watson API error: {parsed_message}")
+ raise Exception(f"IBM Watson API error: {parsed_message}") from e
raise
@@ -535,7 +866,9 @@ async def _test_watsonx_embedding(
if token_response.status_code != 200:
error_details = _extract_error_details(token_response)
- logger.error(f"IBM IAM token request failed: {token_response.status_code} - {error_details}")
+ logger.error(
+ f"IBM IAM token request failed: {token_response.status_code} - {error_details}"
+ )
raise Exception(f"Failed to authenticate with IBM Watson: {error_details}")
bearer_token = token_response.json().get("access_token")
@@ -567,7 +900,9 @@ async def _test_watsonx_embedding(
if response.status_code != 200:
error_details = _extract_error_details(response)
- logger.error(f"IBM Watson embedding test failed: {response.status_code} - {error_details}")
+ logger.error(
+ f"IBM Watson embedding test failed: {response.status_code} - {error_details}"
+ )
# If error_details is still JSON, parse it to extract just the message
parsed_details = _parse_json_error_message(error_details)
raise Exception(f"IBM Watson API error: {parsed_details}")
@@ -580,7 +915,7 @@ async def _test_watsonx_embedding(
except httpx.TimeoutException:
logger.error("IBM Watson embedding test timed out")
- raise Exception("Request timed out")
+ raise Exception("Request timed out") from None
except Exception as e:
logger.error(f"IBM Watson embedding test failed: {str(e)}")
# If the error message contains JSON, parse it to extract just the message
@@ -589,14 +924,14 @@ async def _test_watsonx_embedding(
json_part = error_str.split("IBM Watson API error: ", 1)[1]
parsed_message = _parse_json_error_message(json_part)
if parsed_message != json_part:
- raise Exception(f"IBM Watson API error: {parsed_message}")
+ raise Exception(f"IBM Watson API error: {parsed_message}") from e
raise
# Ollama validation functions
async def _test_ollama_lightweight_health(endpoint: str) -> None:
"""Test Ollama availability with lightweight status check.
-
+
Only checks if the endpoint returns a 200 status without fetching data.
"""
try:
@@ -610,14 +945,16 @@ async def _test_ollama_lightweight_health(endpoint: str) -> None:
if response.status_code != 200:
error_details = _extract_error_details(response)
- logger.error(f"Ollama lightweight health check failed: {response.status_code} - {error_details}")
+ logger.error(
+ f"Ollama lightweight health check failed: {response.status_code} - {error_details}"
+ )
raise Exception(f"Ollama endpoint not responding: {error_details}")
logger.info("Ollama lightweight health check passed")
except httpx.TimeoutException:
logger.error("Ollama lightweight health check timed out")
- raise Exception("Ollama endpoint timed out")
+ raise Exception("Ollama endpoint timed out") from None
except Exception as e:
logger.error(f"Ollama lightweight health check failed: {str(e)}")
raise
@@ -631,9 +968,7 @@ async def _test_ollama_completion_with_tools(llm_model: str, endpoint: str) -> N
payload = {
"model": llm_model,
- "messages": [
- {"role": "user", "content": "What tools do you have available?"}
- ],
+ "messages": [{"role": "user", "content": "What tools do you have available?"}],
"tools": [
{
"type": "function",
@@ -643,14 +978,11 @@ async def _test_ollama_completion_with_tools(llm_model: str, endpoint: str) -> N
"parameters": {
"type": "object",
"properties": {
- "location": {
- "type": "string",
- "description": "The city and state"
- }
+ "location": {"type": "string", "description": "The city and state"}
},
- "required": ["location"]
- }
- }
+ "required": ["location"],
+ },
+ },
}
],
"stream": False,
@@ -665,14 +997,16 @@ async def _test_ollama_completion_with_tools(llm_model: str, endpoint: str) -> N
if response.status_code != 200:
error_details = _extract_error_details(response)
- logger.error(f"Ollama completion test failed: {response.status_code} - {error_details}")
+ logger.error(
+ f"Ollama completion test failed: {response.status_code} - {error_details}"
+ )
raise Exception(f"Ollama API error: {error_details}")
logger.info("Ollama completion with tool calling test passed")
except httpx.TimeoutException:
logger.error("Ollama completion test timed out")
- raise httpx.TimeoutException("Ollama is busy or model inference timed out")
+ raise httpx.TimeoutException("Ollama is busy or model inference timed out") from None
except Exception as e:
logger.error(f"Ollama completion test failed: {str(e)}")
raise
@@ -698,7 +1032,9 @@ async def _test_ollama_embedding(embedding_model: str, endpoint: str) -> None:
if response.status_code != 200:
error_details = _extract_error_details(response)
- logger.error(f"Ollama embedding test failed: {response.status_code} - {error_details}")
+ logger.error(
+ f"Ollama embedding test failed: {response.status_code} - {error_details}"
+ )
raise Exception(f"Ollama API error: {error_details}")
data = response.json()
@@ -709,7 +1045,7 @@ async def _test_ollama_embedding(embedding_model: str, endpoint: str) -> None:
except httpx.TimeoutException:
logger.error("Ollama embedding test timed out")
- raise httpx.TimeoutException("Ollama is busy or embedding generation timed out")
+ raise httpx.TimeoutException("Ollama is busy or embedding generation timed out") from None
except Exception as e:
logger.error(f"Ollama embedding test failed: {str(e)}")
raise
@@ -718,7 +1054,7 @@ async def _test_ollama_embedding(embedding_model: str, endpoint: str) -> None:
# Anthropic validation functions
async def _test_anthropic_lightweight_health(api_key: str) -> None:
"""Test Anthropic API key validity with lightweight check.
-
+
Only checks if the API key is valid without consuming credits.
Uses the /v1/models endpoint which doesn't consume credits.
"""
@@ -738,14 +1074,16 @@ async def _test_anthropic_lightweight_health(api_key: str) -> None:
if response.status_code != 200:
error_details = _extract_error_details(response)
- logger.error(f"Anthropic lightweight health check failed: {response.status_code} - {error_details}")
+ logger.error(
+ f"Anthropic lightweight health check failed: {response.status_code} - {error_details}"
+ )
raise Exception(f"Anthropic API key validation failed: {error_details}")
logger.info("Anthropic lightweight health check passed")
except httpx.TimeoutException:
logger.error("Anthropic lightweight health check timed out")
- raise Exception("Anthropic API request timed out")
+ raise Exception("Anthropic API request timed out") from None
except Exception as e:
logger.error(f"Anthropic lightweight health check failed: {str(e)}")
raise
@@ -764,9 +1102,7 @@ async def _test_anthropic_completion_with_tools(api_key: str, llm_model: str) ->
payload = {
"model": llm_model,
"max_tokens": 50,
- "messages": [
- {"role": "user", "content": "What tools do you have available?"}
- ],
+ "messages": [{"role": "user", "content": "What tools do you have available?"}],
"tools": [
{
"name": "get_weather",
@@ -774,13 +1110,10 @@ async def _test_anthropic_completion_with_tools(api_key: str, llm_model: str) ->
"input_schema": {
"type": "object",
"properties": {
- "location": {
- "type": "string",
- "description": "The city and state"
- }
+ "location": {"type": "string", "description": "The city and state"}
},
- "required": ["location"]
- }
+ "required": ["location"],
+ },
}
],
}
@@ -795,14 +1128,16 @@ async def _test_anthropic_completion_with_tools(api_key: str, llm_model: str) ->
if response.status_code != 200:
error_details = _extract_error_details(response)
- logger.error(f"Anthropic completion test failed: {response.status_code} - {error_details}")
+ logger.error(
+ f"Anthropic completion test failed: {response.status_code} - {error_details}"
+ )
raise Exception(f"Anthropic API error: {error_details}")
logger.info("Anthropic completion with tool calling test passed")
except httpx.TimeoutException:
logger.error("Anthropic completion test timed out")
- raise Exception("Request timed out")
+ raise Exception("Request timed out") from None
except Exception as e:
logger.error(f"Anthropic completion test failed: {str(e)}")
raise
diff --git a/src/services/chat_service.py b/src/services/chat_service.py
index 1b89f7a73..bf9749dd9 100644
--- a/src/services/chat_service.py
+++ b/src/services/chat_service.py
@@ -59,6 +59,7 @@ class ChatService:
user_id: str = None,
jwt_token: str = None,
previous_response_id: str = None,
+ conversation_id: str = None,
stream: bool = False,
filter_id: str = None,
owner: str = None,
@@ -215,6 +216,7 @@ class ChatService:
conversation_user_id,
extra_headers=extra_headers,
previous_response_id=previous_response_id,
+ conversation_id=conversation_id,
filter_id=filter_id,
)
else:
@@ -227,6 +229,7 @@ class ChatService:
conversation_user_id,
extra_headers=extra_headers,
previous_response_id=previous_response_id,
+ conversation_id=conversation_id,
filter_id=filter_id,
)
response_data = {"response": response_text}
@@ -701,20 +704,45 @@ class ChatService:
"total_conversations": len(conversations),
}
+ @staticmethod
+ def _messages_from_active_conversation(conversation_state: dict) -> list[dict]:
+ """Serialize in-memory OpenRAG thread messages for the history API."""
+ messages = []
+ for msg in conversation_state.get("messages", []):
+ if msg.get("role") not in ["user", "assistant"]:
+ continue
+ message_data = {
+ "role": msg["role"],
+ "content": msg["content"],
+ "timestamp": msg.get("timestamp").isoformat()
+ if hasattr(msg.get("timestamp"), "isoformat")
+ else msg.get("timestamp"),
+ "source": "openrag_memory",
+ }
+ if msg.get("error"):
+ message_data["error"] = True
+ if msg.get("chunks"):
+ message_data["chunks"] = msg["chunks"]
+ if msg.get("response_data"):
+ message_data["response_data"] = msg["response_data"]
+ messages.append(message_data)
+ return messages
+
async def get_langflow_history(self, user_id: str):
"""Get langflow conversation history for a user - now fetches from both OpenRAG memory and Langflow database"""
- from agent import get_user_conversations
+ from agent import active_conversations, get_user_conversations
from services.langflow_history_service import langflow_history_service
if not user_id:
return {"error": "User ID is required", "conversations": []}
all_conversations = []
+ local_metadata: dict = {}
+ in_memory = active_conversations.get(user_id, {})
try:
# 1. Get local conversation metadata (no actual messages stored here)
conversations_dict = await get_user_conversations(user_id)
- local_metadata = {}
for response_id, conversation_metadata in conversations_dict.items():
# Store metadata for later use with Langflow data
@@ -756,56 +784,102 @@ class ChatService:
messages.append(message_data)
+ # Prefer OpenRAG memory when it has more turns (error retries use a
+ # fresh Langflow session under the same OpenRAG thread id).
+ memory_messages = self._messages_from_active_conversation(
+ in_memory.get(session_id, {})
+ )
+ if len(memory_messages) > len(messages):
+ messages = memory_messages
+ source = "openrag_memory_preferred"
+ else:
+ source = "langflow_enhanced"
+
if messages:
- # Use local metadata if available, otherwise generate from Langflow data
metadata = local_metadata.get(session_id, {})
-
- if not metadata.get("title"):
- first_user_msg = next(
- (msg for msg in messages if msg["role"] == "user"), None
- )
- title = (
- first_user_msg["content"][:50] + "..."
- if first_user_msg and len(first_user_msg["content"]) > 50
- else first_user_msg["content"]
- if first_user_msg
- else "Langflow chat"
- )
- else:
- title = metadata["title"]
+ first_user_msg = next(
+ (msg for msg in messages if msg["role"] == "user"), None
+ )
+ title = metadata.get("title") or (
+ first_user_msg["content"][:50] + "..."
+ if first_user_msg and len(first_user_msg["content"]) > 50
+ else first_user_msg["content"]
+ if first_user_msg
+ else "Langflow chat"
+ )
all_conversations.append(
{
"response_id": session_id,
"title": title,
"endpoint": "langflow",
- "messages": messages, # Function calls preserved from Langflow
+ "messages": messages,
"created_at": metadata.get("created_at")
or conversation.get("created_at"),
"last_activity": metadata.get("last_activity")
or conversation.get("last_activity"),
"filter_id": metadata.get("filter_id"),
"total_messages": len(messages),
- "source": "langflow_enhanced",
+ "source": source,
"langflow_session_id": session_id,
"langflow_flow_id": conversation.get("flow_id"),
}
)
- if langflow_history.get("conversations"):
- logger.debug(
- f"Added {len(langflow_history['conversations'])} historical conversations from Langflow"
- )
- elif langflow_history.get("error"):
- logger.debug(
- f"Could not fetch Langflow history for user {user_id}: {langflow_history['error']}"
- )
- else:
- logger.debug(f"No Langflow conversations found for user {user_id}")
-
except Exception as e:
logger.error(f"Failed to fetch Langflow history: {e}")
- # Continue with just in-memory conversations
+
+ # Local-only threads + upgrade listed ones from memory/metadata snapshots
+ # (Langflow history is incomplete after fresh-session error retries).
+ listed_ids = {c["response_id"] for c in all_conversations}
+ for response_id, metadata in local_metadata.items():
+ conversation_state = in_memory.get(response_id, {})
+ messages = self._messages_from_active_conversation(conversation_state)
+ if not messages and isinstance(metadata.get("messages"), list):
+ messages = [
+ m
+ for m in metadata["messages"]
+ if isinstance(m, dict) and m.get("role") in ["user", "assistant"]
+ ]
+
+ if response_id in listed_ids:
+ existing = next(c for c in all_conversations if c["response_id"] == response_id)
+ if len(messages) > len(existing.get("messages") or []):
+ existing["messages"] = messages
+ existing["total_messages"] = len(messages)
+ existing["source"] = "openrag_memory_preferred"
+ continue
+
+ if not messages and not metadata.get("title"):
+ continue
+
+ first_user_msg = next((msg for msg in messages if msg["role"] == "user"), None)
+ title = metadata.get("title") or (
+ first_user_msg["content"][:50] + "..."
+ if first_user_msg and len(first_user_msg["content"]) > 50
+ else first_user_msg["content"]
+ if first_user_msg
+ else "New Chat"
+ )
+ created_at = metadata.get("created_at") or conversation_state.get("created_at")
+ last_activity = metadata.get("last_activity") or conversation_state.get("last_activity")
+ all_conversations.append(
+ {
+ "response_id": response_id,
+ "title": title,
+ "endpoint": "langflow",
+ "messages": messages,
+ "created_at": created_at.isoformat()
+ if hasattr(created_at, "isoformat")
+ else created_at,
+ "last_activity": last_activity.isoformat()
+ if hasattr(last_activity, "isoformat")
+ else last_activity,
+ "filter_id": metadata.get("filter_id") or conversation_state.get("filter_id"),
+ "total_messages": len(messages) or metadata.get("total_messages", 0),
+ "source": "local_error_fallback",
+ }
+ )
# Sort by last activity (most recent first)
all_conversations.sort(key=lambda c: c.get("last_activity", ""), reverse=True)
diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.py
index ff560aca7..48e36330e 100644
--- a/src/services/langflow_file_service.py
+++ b/src/services/langflow_file_service.py
@@ -4,7 +4,7 @@ import json
import time
import uuid
from pathlib import Path
-from typing import Any
+from typing import Any, NoReturn
import httpx
@@ -307,6 +307,15 @@ class LangflowFileService:
error=str(e),
)
+ async def _raise_resolved_ingest_error(self, exc: BaseException) -> NoReturn:
+ """Re-raise with a credential message when Langflow disconnects on bad API keys."""
+ from api.provider_validation import resolve_ingest_error_message
+
+ resolved = await resolve_ingest_error_message(exc)
+ if resolved != (str(exc) or "").strip():
+ raise Exception(resolved) from exc
+ raise exc
+
async def upload_user_file(self, file_tuple, jwt_token: str | None = None) -> dict[str, Any]:
"""Upload a file using Langflow Files API v2: POST /api/v2/files.
Returns JSON with keys: id, name, path, size, provider.
@@ -589,12 +598,12 @@ class LangflowFileService:
raise
return resp_json
- except Exception:
+ except Exception as e:
await self._cleanup_failed_callback_ingest(
ingest_token=ingest_token,
ingest_run_id=ingest_run_id,
)
- raise
+ await self._raise_resolved_ingest_error(e)
async def run_url_ingestion_flow(
self,
@@ -724,12 +733,12 @@ class LangflowFileService:
)
return resp.json()
- except Exception:
+ except Exception as e:
await self._cleanup_failed_callback_ingest(
ingest_token=ingest_token,
ingest_run_id=ingest_run_id,
)
- raise
+ await self._raise_resolved_ingest_error(e)
async def _ensure_url_ingest_flow_id(self) -> str:
"""Ensure URL ingest flow ID is valid; import flow if missing.
diff --git a/src/services/langflow_history_service.py b/src/services/langflow_history_service.py
index dc4111486..35a2405cc 100644
--- a/src/services/langflow_history_service.py
+++ b/src/services/langflow_history_service.py
@@ -3,32 +3,35 @@ Langflow Message History Service
Simplified service that retrieves message history from Langflow using shared client infrastructure
"""
-from typing import List, Dict, Optional, Any
+from typing import Any
+from api.provider_validation import (
+ looks_like_provider_error_content,
+ sanitize_provider_error_content,
+)
from config.settings import clients
from utils.logging_config import get_logger
logger = get_logger(__name__)
+
class LangflowHistoryService:
"""Simplified service to retrieve message history from Langflow"""
-
+
def __init__(self):
pass
-
- async def get_user_sessions(self, user_id: str, flow_id: Optional[str] = None) -> List[str]:
+
+ async def get_user_sessions(self, user_id: str, flow_id: str | None = None) -> list[str]:
"""Get all session IDs for a user's conversations"""
try:
params = {}
if flow_id:
params["flow_id"] = flow_id
-
+
response = await clients.langflow_request(
- "GET",
- "/api/v1/monitor/messages/sessions",
- params=params
+ "GET", "/api/v1/monitor/messages/sessions", params=params
)
-
+
if response.status_code == 200:
session_ids = response.json()
logger.debug(f"Found {len(session_ids)} total sessions from Langflow")
@@ -36,65 +39,73 @@ class LangflowHistoryService:
else:
logger.error(f"Failed to get sessions: {response.status_code} - {response.text}")
return []
-
+
except Exception as e:
logger.error(f"Error getting user sessions: {e}")
return []
-
- async def get_session_messages(self, user_id: str, session_id: str) -> List[Dict[str, Any]]:
+
+ async def get_session_messages(self, user_id: str, session_id: str) -> list[dict[str, Any]]:
"""Get all messages for a specific session"""
try:
response = await clients.langflow_request(
"GET",
"/api/v1/monitor/messages",
- params={
- "session_id": session_id,
- "order_by": "timestamp"
- }
+ params={"session_id": session_id, "order_by": "timestamp"},
)
-
+
if response.status_code == 200:
messages = response.json()
# Convert to OpenRAG format
return self._convert_langflow_messages(messages)
else:
- logger.error(f"Failed to get messages for session {session_id}: {response.status_code}")
+ logger.error(
+ f"Failed to get messages for session {session_id}: {response.status_code}"
+ )
return []
-
+
except Exception as e:
logger.error(f"Error getting session messages: {e}")
return []
-
- def _convert_langflow_messages(self, langflow_messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+
+ def _convert_langflow_messages(
+ self, langflow_messages: list[dict[str, Any]]
+ ) -> list[dict[str, Any]]:
"""Convert Langflow messages to OpenRAG format"""
- converted_messages = []
-
+ converted_messages: list[dict[str, Any]] = []
+
for msg in langflow_messages:
try:
- content = msg.get("text", "")
-
- # Detect error messages - check explicit error flag or common error patterns
- is_error = msg.get("error", False) or (
- msg.get("sender") != "User" and content.startswith("Error:")
- )
-
+ content = msg.get("text", "") or ""
+ sender = msg.get("sender")
+ is_user = sender == "User"
+
+ # Detect error messages - explicit flag, Error: prefix, or provider failures
+ # (including raw JSON payloads that leak from upstream auth/rate-limit errors).
+ is_error = False
+ if not is_user:
+ is_error = bool(msg.get("error", False)) or looks_like_provider_error_content(
+ content
+ )
+ if is_error:
+ content = sanitize_provider_error_content(content)
+
# Map Langflow message format to OpenRAG format
converted_msg = {
- "role": "user" if msg.get("sender") == "User" else "assistant",
+ "role": "user" if is_user else "assistant",
"content": content,
"timestamp": msg.get("timestamp"),
"langflow_message_id": msg.get("id"),
"langflow_session_id": msg.get("session_id"),
"langflow_flow_id": msg.get("flow_id"),
- "sender": msg.get("sender"),
+ "sender": sender,
"sender_name": msg.get("sender_name"),
"source": "langflow",
"files": msg.get("files", []),
"properties": msg.get("properties", {}),
"error": is_error,
- "edit": msg.get("edit", False)
+ "edit": msg.get("edit", False),
}
-
+
# Extract function calls from content_blocks if present
# Convert to match streaming format: chunk.item.type === "tool_call"
content_blocks = msg.get("content_blocks", [])
@@ -102,43 +113,59 @@ class LangflowHistoryService:
chunks = []
for block in content_blocks:
if block.get("title") == "Agent Steps" and block.get("contents"):
- for content in block["contents"]:
- if content.get("type") == "tool_use":
+ for content_item in block["contents"]:
+ if content_item.get("type") == "tool_use":
# Convert Langflow tool_use format to match streaming chunks format
# Frontend expects: chunk.item.type === "tool_call" with tool_name, inputs, results
chunk = {
"type": "response.output_item.added",
"item": {
"type": "tool_call",
- "tool_name": content.get("name", ""),
- "inputs": content.get("tool_input", {}),
- "results": content.get("output", {}),
- "id": content.get("id") or content.get("run_id", ""),
- "status": "completed" if not content.get("error") else "error"
- }
+ "tool_name": content_item.get("name", ""),
+ "inputs": content_item.get("tool_input", {}),
+ "results": content_item.get("output", {}),
+ "id": content_item.get("id")
+ or content_item.get("run_id", ""),
+ "status": "completed"
+ if not content_item.get("error")
+ else "error",
+ },
}
chunks.append(chunk)
-
+
if chunks:
converted_msg["chunks"] = chunks
-
+
+ # Langflow often records the same provider failure several times
+ # in one session (e.g. embedding + LLM auth retries). Keep one.
+ prev = converted_messages[-1] if converted_messages else None
+ if (
+ is_error
+ and prev
+ and prev.get("error")
+ and (prev.get("content") or "").strip() == content.strip()
+ ):
+ continue
+
converted_messages.append(converted_msg)
-
+
except Exception as e:
logger.error(f"Error converting message: {e}")
continue
-
+
return converted_messages
-
- async def get_user_conversation_history(self, user_id: str, flow_id: Optional[str] = None) -> Dict[str, Any]:
+
+ async def get_user_conversation_history(
+ self, user_id: str, flow_id: str | None = None
+ ) -> dict[str, Any]:
"""Get all conversation history for a user, organized by session
-
+
Simplified version - gets all sessions and lets the frontend filter by user_id
"""
try:
# Get all sessions (no complex filtering needed)
session_ids = await self.get_user_sessions(user_id, flow_id)
-
+
conversations = []
for session_id in session_ids:
messages = await self.get_session_messages(user_id, session_id)
@@ -146,7 +173,7 @@ class LangflowHistoryService:
# Create conversation metadata
first_message = messages[0] if messages else None
last_message = messages[-1] if messages else None
-
+
conversation = {
"session_id": session_id,
"langflow_session_id": session_id, # For compatibility
@@ -156,26 +183,24 @@ class LangflowHistoryService:
"created_at": first_message.get("timestamp") if first_message else None,
"last_activity": last_message.get("timestamp") if last_message else None,
"flow_id": first_message.get("langflow_flow_id") if first_message else None,
- "source": "langflow"
+ "source": "langflow",
}
conversations.append(conversation)
-
- # Sort by last activity (most recent first)
- conversations.sort(key=lambda c: c.get("last_activity", ""), reverse=True)
-
+
+ # Sort by last activity (most recent first). Normalize None so mixed
+ # None/str timestamps do not raise TypeError during comparison.
+ conversations.sort(key=lambda c: c.get("last_activity") or "", reverse=True)
+
return {
"conversations": conversations,
"total_conversations": len(conversations),
- "user_id": user_id
+ "user_id": user_id,
}
-
+
except Exception as e:
logger.error(f"Error getting user conversation history: {e}")
- return {
- "error": str(e),
- "conversations": []
- }
+ return {"error": str(e), "conversations": []}
# Global instance
-langflow_history_service = LangflowHistoryService()
\ No newline at end of file
+langflow_history_service = LangflowHistoryService()
diff --git a/src/services/task_service.py b/src/services/task_service.py
index 78e7e7002..7739d868e 100644
--- a/src/services/task_service.py
+++ b/src/services/task_service.py
@@ -91,6 +91,28 @@ def _is_task_cancellation_error(error: str) -> bool:
return any(marker in lowered for marker in _TASK_CANCELLATION_ERROR_MARKERS)
+def _provider_credential_failure_metadata(error: str) -> dict | None:
+ """Classify invalid/revoked provider API key failures for ingestion toasts/UI."""
+ if not error:
+ return None
+
+ from api.provider_validation import (
+ is_provider_credential_error,
+ sanitize_provider_error_content,
+ )
+
+ cleaned = sanitize_provider_error_content(error)
+ if not (is_provider_credential_error(error) or is_provider_credential_error(cleaned)):
+ return None
+
+ return {
+ "component": "openrag",
+ "failure_phase": "embedding",
+ "user_facing_message": cleaned,
+ "actionable_by": "USER_ACTIONABLE",
+ }
+
+
def _is_transient_connectivity_error(error: str) -> bool:
lowered = error.lower()
return any(marker in lowered for marker in _TRANSIENT_CONNECTIVITY_ERROR_MARKERS)
@@ -1088,6 +1110,14 @@ class TaskService:
"actionable_by": "USER_ACTIONABLE",
}
+ # Prefer credential failures over transport messages. Langflow may report
+ # "server disconnected" when the real cause is a revoked embedding API key;
+ # resolve_ingest_error_message() rewrites file_task.error in that case, and
+ # this check also covers errors that mention both disconnect + API key text.
+ credential_meta = _provider_credential_failure_metadata(error)
+ if credential_meta:
+ return credential_meta
+
if _is_transient_connectivity_error(error) or _is_langflow_transport_failure(error):
return {
"component": "langflow",
diff --git a/tests/unit/test_langflow_stream_provider_errors.py b/tests/unit/test_langflow_stream_provider_errors.py
new file mode 100644
index 000000000..15fa3c504
--- /dev/null
+++ b/tests/unit/test_langflow_stream_provider_errors.py
@@ -0,0 +1,453 @@
+"""Provider errors must reach the client as a final stream error chunk.
+
+These tests mock the upstream stream to raise exceptions shaped like real
+provider failures (watsonx rate limit, invalid key, permission). That proves
+the yield path without hardcoding a raise in production code.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import AsyncIterator
+
+import pytest
+
+import agent as agent_module
+
+
+def test_format_provider_error_message_extracts_embedded_json():
+ raw = (
+ 'Provider request failed: {"error": {"message": '
+ '"Incorrect API key provided", "type": "invalid_request_error"}}'
+ )
+ assert (
+ agent_module._format_provider_error_message(Exception(raw)) == "Incorrect API key provided"
+ )
+
+
+def test_format_provider_error_message_extracts_ibm_iam_error_message():
+ raw = (
+ 'Failed to authenticate with IBM Watson: {"errorCode":"BXNIM0415E",'
+ '"errorMessage":"Provided API key could not be found.",'
+ '"context":{"requestId":"abc","url":"https://iam.cloud.ibm.com"}}'
+ )
+ assert agent_module._format_provider_error_message(Exception(raw)) == (
+ "Provided API key could not be found."
+ )
+
+
+def test_format_provider_error_message_keeps_plain_text():
+ msg = "Rate limit exceeded for watsonx.ai. Please try again later."
+ assert agent_module._format_provider_error_message(Exception(msg)) == msg
+
+
+def test_provider_error_display_text_keeps_partial_answer():
+ assert agent_module._provider_error_display_text(
+ "Rate limit exceeded", "Partial answer so far"
+ ) == ("Partial answer so far\n\nRate limit exceeded")
+
+
+def test_provider_error_display_text_drops_error_like_partial():
+ partial = (
+ "Failed to initialize IBM WatsonX embedding model. Error: "
+ '{"errorMessage":"Provided API key could not be found."}'
+ )
+ assert (
+ agent_module._provider_error_display_text("Provided API key could not be found.", partial)
+ == "Provided API key could not be found."
+ )
+
+
+def test_resolve_conversation_store_id_prefers_warm_thread():
+ cold_id = agent_module._resolve_conversation_store_id(
+ None,
+ "resp_cold",
+ previous_loaded_from_memory=False,
+ mint_if_missing=True,
+ )
+ # Cold previous ids must not be reused (would overwrite metadata); mint a new id.
+ assert cold_id != "resp_cold"
+ assert not cold_id.startswith("error_")
+
+ assert (
+ agent_module._resolve_conversation_store_id(
+ None,
+ "resp_warm",
+ previous_loaded_from_memory=True,
+ )
+ == "resp_warm"
+ )
+ assert (
+ agent_module._resolve_conversation_store_id(
+ "resp_stream",
+ "resp_warm",
+ previous_loaded_from_memory=True,
+ )
+ == "resp_warm"
+ )
+
+
+def store_in_memory(monkeypatch):
+ stored: list[tuple[str, str]] = []
+ claimed: list[tuple[str, str]] = []
+
+ async def _store(user_id, response_id, conversation_state):
+ stored.append((user_id, response_id))
+ if user_id not in agent_module.active_conversations:
+ agent_module.active_conversations[user_id] = {}
+ agent_module.active_conversations[user_id][response_id] = conversation_state
+
+ class _Ownership:
+ async def claim_session(self, user_id, session_id):
+ claimed.append((user_id, session_id))
+
+ monkeypatch.setattr(agent_module, "store_conversation_thread", _store)
+ monkeypatch.setattr(
+ "services.session_ownership_service.session_ownership_service",
+ _Ownership(),
+ )
+ return {"stored": stored, "claimed": claimed}
+
+
+async def _collect_error_chunks(stream) -> list[dict]:
+ chunks: list[dict] = []
+ async for raw in stream:
+ chunks.append(json.loads(raw.decode("utf-8")))
+ return chunks
+
+
+ "provider_message",
+ [
+ "Rate limit exceeded for watsonx.ai. Please try again later.",
+ "Invalid API key for Anthropic. Check your credentials.",
+ "Permission denied: model access not authorized for this project.",
+ "Quota exceeded: you have reached your monthly token limit.",
+ ],
+)
+async def test_langflow_stream_yields_provider_error_chunk(
+ monkeypatch, store_in_memory, provider_message: str
+):
+ async def raise_provider_error(*_args, **_kwargs) -> AsyncIterator[bytes]:
+ raise Exception(provider_message)
+ yield b"" # pragma: no cover — keep this an async generator
+
+ monkeypatch.setattr(agent_module, "async_stream", raise_provider_error)
+
+ user_id = "test-provider-error-user"
+ agent_module.active_conversations.pop(user_id, None)
+
+ chunks = await _collect_error_chunks(
+ agent_module.async_langflow_chat_stream(
+ langflow_client=object(),
+ flow_id="flow-id",
+ prompt="hello",
+ user_id=user_id,
+ )
+ )
+
+ assert len(chunks) == 1
+ error_chunk = chunks[0]
+ assert error_chunk["status"] == "failed"
+ assert error_chunk["finish_reason"] == "error"
+ assert error_chunk["error"]["message"] == provider_message
+
+ # Failed first turns still get a real store id (UUID, not error_*).
+ assert len(store_in_memory["stored"]) == 1
+ stored_user, stored_id = store_in_memory["stored"][0]
+ assert stored_user == user_id
+ assert not stored_id.startswith("error_")
+ assert error_chunk.get("id") == stored_id
+ assert stored_id in agent_module.active_conversations.get(user_id, {})
+
+
+async def test_langflow_stream_persists_error_onto_existing_thread(monkeypatch, store_in_memory):
+ provider_message = "Invalid API key for Anthropic. Check your credentials."
+
+ async def raise_provider_error(*_args, **_kwargs) -> AsyncIterator[bytes]:
+ raise Exception(provider_message)
+ yield b"" # pragma: no cover
+
+ monkeypatch.setattr(agent_module, "async_stream", raise_provider_error)
+
+ user_id = "test-provider-error-existing"
+ previous_id = "resp_existing_thread"
+ agent_module.active_conversations[user_id] = {
+ previous_id: {
+ "messages": [{"role": "user", "content": "earlier"}],
+ "created_at": None,
+ "last_activity": None,
+ }
+ }
+
+ chunks = await _collect_error_chunks(
+ agent_module.async_langflow_chat_stream(
+ langflow_client=object(),
+ flow_id="flow-id",
+ prompt="hello",
+ user_id=user_id,
+ previous_response_id=previous_id,
+ )
+ )
+
+ assert chunks[0]["error"]["message"] == provider_message
+ assert store_in_memory["stored"] == [(user_id, previous_id)]
+ assert store_in_memory["claimed"] == [(user_id, previous_id)]
+ assert not any(rid.startswith("error_") for _, rid in store_in_memory["stored"])
+
+ stored = agent_module.active_conversations[user_id][previous_id]
+ error_messages = [m for m in stored["messages"] if m.get("error")]
+ assert len(error_messages) == 1
+ assert error_messages[0]["content"] == provider_message
+
+
+async def test_langflow_follow_up_after_error_reuses_store_id(monkeypatch, store_in_memory):
+ """Retrying in the same chat must not create a second sidebar conversation."""
+ provider_message = "Rate limit exceeded for watsonx.ai."
+ stream_calls: list[dict] = []
+
+ async def raise_provider_error(*_args, **kwargs) -> AsyncIterator[bytes]:
+ stream_calls.append(kwargs)
+ raise Exception(provider_message)
+ yield b"" # pragma: no cover
+
+ monkeypatch.setattr(agent_module, "async_stream", raise_provider_error)
+
+ user_id = "test-provider-error-follow-up"
+ existing_id = "resp_failed_thread"
+ agent_module.active_conversations[user_id] = {
+ existing_id: {
+ "messages": [
+ {"role": "user", "content": "first"},
+ {
+ "role": "assistant",
+ "content": "Provided API key could not be found.",
+ "error": True,
+ },
+ ],
+ "created_at": None,
+ "last_activity": None,
+ }
+ }
+
+ # Client keeps conversation_id after an error but omits previous_response_id
+ # so Langflow starts fresh while OpenRAG appends to the same sidebar thread.
+ chunks = await _collect_error_chunks(
+ agent_module.async_langflow_chat_stream(
+ langflow_client=object(),
+ flow_id="flow-id",
+ prompt="retry hello",
+ user_id=user_id,
+ conversation_id=existing_id,
+ previous_response_id=None,
+ )
+ )
+
+ assert chunks[0]["error"]["message"] == provider_message
+ assert store_in_memory["stored"] == [(user_id, existing_id)]
+ assert list(agent_module.active_conversations[user_id].keys()) == [existing_id]
+ # Fresh Langflow session, same OpenRAG conversation id.
+ assert stream_calls[0].get("previous_response_id") is None
+ stored_messages = agent_module.active_conversations[user_id][existing_id]["messages"]
+ assert [m["role"] for m in stored_messages if m["role"] != "system"][-2:] == [
+ "user",
+ "assistant",
+ ]
+
+
+async def test_langflow_follow_up_ignores_broken_previous_response_id(monkeypatch, store_in_memory):
+ """Even if the client still sends a failed Langflow session id, do not chain it."""
+ provider_message = "Invalid API key for Anthropic. Check your credentials."
+ stream_calls: list[dict] = []
+
+ async def raise_provider_error(*_args, **kwargs) -> AsyncIterator[bytes]:
+ stream_calls.append(kwargs)
+ raise Exception(provider_message)
+ yield b"" # pragma: no cover
+
+ monkeypatch.setattr(agent_module, "async_stream", raise_provider_error)
+
+ user_id = "test-provider-error-broken-session"
+ existing_id = "resp_broken_langflow"
+ agent_module.active_conversations[user_id] = {
+ existing_id: {
+ "messages": [
+ {"role": "user", "content": "first"},
+ {
+ "role": "assistant",
+ "content": provider_message,
+ "error": True,
+ },
+ ],
+ "created_at": None,
+ "last_activity": None,
+ }
+ }
+
+ chunks = await _collect_error_chunks(
+ agent_module.async_langflow_chat_stream(
+ langflow_client=object(),
+ flow_id="flow-id",
+ prompt="retry hello",
+ user_id=user_id,
+ conversation_id=existing_id,
+ previous_response_id=existing_id,
+ )
+ )
+
+ assert chunks[0]["error"]["message"] == provider_message
+ assert stream_calls[0].get("previous_response_id") is None
+ assert store_in_memory["stored"] == [(user_id, existing_id)]
+
+
+async def test_langflow_stream_skips_persist_for_cold_previous_response_id(
+ monkeypatch, store_in_memory
+):
+ provider_message = "Permission denied: model access not authorized."
+
+ async def raise_provider_error(*_args, **_kwargs) -> AsyncIterator[bytes]:
+ raise Exception(provider_message)
+ yield b"" # pragma: no cover
+
+ monkeypatch.setattr(agent_module, "async_stream", raise_provider_error)
+
+ user_id = "test-provider-error-cold"
+ previous_id = "resp_not_in_memory"
+ agent_module.active_conversations.pop(user_id, None)
+
+ chunks = await _collect_error_chunks(
+ agent_module.async_langflow_chat_stream(
+ langflow_client=object(),
+ flow_id="flow-id",
+ prompt="hello",
+ user_id=user_id,
+ previous_response_id=previous_id,
+ )
+ )
+
+ assert chunks[0]["error"]["message"] == provider_message
+ # Cold previous_response_id must not overwrite conversation metadata —
+ # persist under a freshly minted id instead.
+ assert len(store_in_memory["stored"]) == 1
+ stored_user, stored_id = store_in_memory["stored"][0]
+ assert stored_user == user_id
+ assert stored_id != previous_id
+ assert not stored_id.startswith("error_")
+ assert previous_id not in agent_module.active_conversations.get(user_id, {})
+ assert stored_id in agent_module.active_conversations.get(user_id, {})
+
+
+async def test_langflow_stream_treats_credential_dump_content_as_error(
+ monkeypatch, store_in_memory
+):
+ """WatsonX failures are often streamed as assistant text with no exception."""
+ dump = (
+ "Failed to initialize IBM WatsonX embedding model: Attempt of authenticating "
+ "connection to service failed, please validate your credentials. Error: "
+ '{"errorCode":"BXNIM0415E","errorMessage":"Provided API key could not be found."} '
+ "IBM WatsonX requires additional configuration parameters "
+ "(API endpoint URL and project ID). An error occurred while generating a response."
+ )
+
+ async def stream_dump(*_args, **_kwargs) -> AsyncIterator[bytes]:
+ yield (json.dumps({"id": "resp_dump", "delta": {"content": dump}}) + "\n").encode("utf-8")
+
+ monkeypatch.setattr(agent_module, "async_stream", stream_dump)
+
+ user_id = "test-provider-error-content-dump"
+ agent_module.active_conversations.pop(user_id, None)
+
+ chunks = await _collect_error_chunks(
+ agent_module.async_langflow_chat_stream(
+ langflow_client=object(),
+ flow_id="flow-id",
+ prompt="hello",
+ user_id=user_id,
+ )
+ )
+
+ assert any(c.get("status") == "failed" for c in chunks)
+ error_chunk = next(c for c in chunks if c.get("status") == "failed")
+ assert "{" not in error_chunk["error"]["message"]
+ assert "Provided API key could not be found" in error_chunk["error"]["message"]
+ assert store_in_memory["stored"] == [(user_id, "resp_dump")]
+ assert store_in_memory["claimed"] == [(user_id, "resp_dump")]
+ stored = agent_module.active_conversations[user_id]["resp_dump"]
+ assert stored["messages"][-1]["error"] is True
+ assert "{" not in stored["messages"][-1]["content"]
+
+
+async def test_langflow_stream_keeps_partial_text_with_provider_error(monkeypatch, store_in_memory):
+ provider_message = "Rate limit exceeded for watsonx.ai."
+
+ async def partial_then_fail(*_args, **_kwargs) -> AsyncIterator[bytes]:
+ yield (
+ json.dumps(
+ {
+ "id": "resp_partial",
+ "delta": {"content": "Here is a partial answer"},
+ }
+ )
+ + "\n"
+ ).encode("utf-8")
+ raise Exception(provider_message)
+
+ monkeypatch.setattr(agent_module, "async_stream", partial_then_fail)
+
+ user_id = "test-provider-error-partial"
+ agent_module.active_conversations.pop(user_id, None)
+
+ chunks = await _collect_error_chunks(
+ agent_module.async_langflow_chat_stream(
+ langflow_client=object(),
+ flow_id="flow-id",
+ prompt="hello",
+ user_id=user_id,
+ )
+ )
+
+ assert len(chunks) == 2
+ assert chunks[0]["delta"]["content"] == "Here is a partial answer"
+ assert chunks[1]["error"]["message"] == (f"Here is a partial answer\n\n{provider_message}")
+ assert store_in_memory["stored"] == [(user_id, "resp_partial")]
+ assert store_in_memory["claimed"] == [(user_id, "resp_partial")]
+
+
+async def test_chat_stream_yields_provider_error_chunk(monkeypatch, store_in_memory):
+ provider_message = "Permission denied: model access not authorized for this project."
+
+ async def raise_provider_error(*_args, **_kwargs) -> AsyncIterator[bytes]:
+ raise Exception(provider_message)
+ yield b"" # pragma: no cover
+
+ monkeypatch.setattr(agent_module, "async_stream", raise_provider_error)
+
+ user_id = "test-chat-provider-error-user"
+ agent_module.active_conversations.pop(user_id, None)
+
+ chunks = await _collect_error_chunks(
+ agent_module.async_chat_stream(
+ async_client=object(),
+ prompt="hello",
+ user_id=user_id,
+ model="gpt-4.1-mini",
+ )
+ )
+
+ assert len(chunks) == 1
+ assert chunks[0]["error"]["message"] == provider_message
+ assert len(store_in_memory["stored"]) == 1
+ stored_user, stored_id = store_in_memory["stored"][0]
+ assert stored_user == user_id
+ assert not stored_id.startswith("error_")
+ assert chunks[0].get("id") == stored_id
+ assert stored_id in agent_module.active_conversations.get(user_id, {})
diff --git a/tests/unit/test_provider_error_formatting.py b/tests/unit/test_provider_error_formatting.py
new file mode 100644
index 000000000..4a62f3716
--- /dev/null
+++ b/tests/unit/test_provider_error_formatting.py
@@ -0,0 +1,171 @@
+"""Unit tests for provider error message formatting helpers."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from api.provider_validation import (
+ format_provider_error_message,
+ is_generic_upstream_error,
+ is_provider_credential_error,
+ looks_like_provider_error_content,
+ resolve_chat_stream_error_message,
+ sanitize_provider_error_content,
+)
+
+
+def test_parse_ibm_iam_error_message():
+ raw = (
+ 'Failed to authenticate with IBM Watson: {"errorCode":"BXNIM0415E",'
+ '"errorMessage":"Provided API key could not be found.",'
+ '"context":{"requestId":"abc","url":"https://iam.cloud.ibm.com"}}'
+ )
+ assert format_provider_error_message(raw) == "Provided API key could not be found."
+
+
+def test_format_extracts_json_with_trailing_text():
+ raw = (
+ "Failed to initialize IBM WatsonX embedding model: Attempt of authenticating "
+ "connection to service failed, please validate your credentials. Error: "
+ '{"errorCode":"BXNIM0415E","errorMessage":"Provided API key could not be found.",'
+ '"context":{"requestId":"abc"}} '
+ "IBM WatsonX requires additional configuration parameters. "
+ "An error occurred while generating a response."
+ )
+ assert format_provider_error_message(raw) == "Provided API key could not be found."
+ assert "{" not in sanitize_provider_error_content(raw)
+
+
+def test_is_provider_credential_error():
+ assert is_provider_credential_error("Incorrect API key provided")
+ assert is_provider_credential_error("Provided API key could not be found.")
+ assert is_provider_credential_error("Provided API key is disabled.")
+ assert is_provider_credential_error(json.dumps({"errorMessage": "api key revoked"}))
+ assert not is_provider_credential_error("Rate limit exceeded")
+
+
+def test_disabled_watsonx_key_embedding_dump_is_credential_error():
+ """Langflow embedding failures for disabled keys omit 'failed to authenticate'."""
+ raw = (
+ "Error running graph: Error building Component Embedding Model: "
+ "Failed to initialize IBM WatsonX embedding model: Attempt of authenticating "
+ "connection to service failed, please validate your credentials. Error: "
+ '{"errorCode":"BXNIM0420E","errorMessage":"Provided API key is disabled."}'
+ )
+ cleaned = sanitize_provider_error_content(raw)
+ assert is_provider_credential_error(raw) or is_provider_credential_error(cleaned)
+ assert cleaned == "Provided API key is disabled."
+ assert "{" not in cleaned
+
+
+def test_strip_error_label_prefixes_without_json():
+ assert (
+ format_provider_error_message(
+ "Error running graph: Error building Component Language Model: Rate limit exceeded"
+ )
+ == "Rate limit exceeded"
+ )
+ # Bare "Error: …" is preserved (no label after Error).
+ assert format_provider_error_message("Error: boom") == "Error: boom"
+
+
+def test_looks_like_provider_error_content():
+ assert looks_like_provider_error_content("Error: boom")
+ assert looks_like_provider_error_content("An unknown error occurred.")
+ assert looks_like_provider_error_content(
+ 'Failed to authenticate: {"errorMessage":"Provided API key could not be found."}'
+ )
+ assert not looks_like_provider_error_content("OpenRAG is an open-source package.")
+ assert is_generic_upstream_error("An unknown error occurred.")
+
+
+def test_resolve_chat_stream_error_message_keeps_generic_without_probing():
+ # Opaque upstream text must not be rewritten into an inferred API-key error.
+ assert (
+ resolve_chat_stream_error_message("An unknown error occurred.")
+ == "An unknown error occurred."
+ )
+
+
+async def test_resolve_ingest_error_message_probes_credentials_on_disconnect(monkeypatch):
+ async def fake_probe():
+ return "Provided API key could not be found."
+
+ monkeypatch.setattr(
+ "api.provider_validation.probe_provider_credential_error",
+ fake_probe,
+ )
+
+ from api.provider_validation import resolve_ingest_error_message
+
+ assert (
+ await resolve_ingest_error_message("Server disconnected without sending a response.")
+ == "Provided API key could not be found."
+ )
+
+
+async def test_resolve_ingest_error_message_keeps_disconnect_when_probe_clean(monkeypatch):
+ async def fake_probe():
+ return None
+
+ monkeypatch.setattr(
+ "api.provider_validation.probe_provider_credential_error",
+ fake_probe,
+ )
+
+ from api.provider_validation import resolve_ingest_error_message
+
+ raw = "Server disconnected without sending a response."
+ assert await resolve_ingest_error_message(raw) == raw
+
+
+async def test_probe_checks_non_selected_providers_with_keys(monkeypatch):
+ """Revoked watsonx must be detected even when openai is the selected provider."""
+
+ class FakeProvider:
+ def __init__(self, api_key="", endpoint=None, project_id=None):
+ self.api_key = api_key
+ self.endpoint = endpoint
+ self.project_id = project_id
+
+ class FakeConfig:
+ class knowledge:
+ embedding_provider = "openai"
+ embedding_model = "text-embedding-3-small"
+
+ class agent:
+ llm_provider = "openai"
+ llm_model = "gpt-4o-mini"
+
+ class providers:
+ openai = FakeProvider(api_key="sk-ok")
+ anthropic = FakeProvider()
+ watsonx = FakeProvider(api_key="bad-watsonx", project_id="proj")
+ ollama = FakeProvider()
+
+ def get_embedding_provider_config(self):
+ return self.providers.openai
+
+ def get_llm_provider_config(self):
+ return self.providers.openai
+
+ async def fake_validate(**kwargs):
+ if kwargs.get("provider") == "watsonx":
+ raise Exception(
+ "Failed to authenticate with IBM Watson: Provided API key could not be found."
+ )
+
+ monkeypatch.setattr("config.settings.get_openrag_config", lambda: FakeConfig())
+ monkeypatch.setattr("api.provider_validation.validate_provider_setup", fake_validate)
+
+ from api.provider_validation import probe_provider_credential_error
+
+ assert (
+ await probe_provider_credential_error()
+ == "Failed to authenticate with IBM Watson: Provided API key could not be found."
+ )
diff --git a/tests/unit/test_task_service_get_task_status2.py b/tests/unit/test_task_service_get_task_status2.py
index 64660959f..cbb76008a 100644
--- a/tests/unit/test_task_service_get_task_status2.py
+++ b/tests/unit/test_task_service_get_task_status2.py
@@ -170,6 +170,58 @@ class TestInferFailureMetadata:
assert meta["failure_phase"] == "unknown"
assert meta["actionable_by"] == "RETRYABLE"
+ def test_langflow_revoked_api_key_failure(self, task_service):
+ ft = _make_file_task(
+ phase=IngestionPhase.LANGFLOW,
+ docling_status=DoclingPhaseStatus.SUCCESS,
+ error=(
+ 'Failed to authenticate with IBM Watson: {"errorCode":"BXNIM0415E",'
+ '"errorMessage":"Provided API key could not be found."}'
+ ),
+ )
+ meta = task_service._infer_failure_metadata(ft)
+ assert meta is not None
+ assert meta["component"] == "openrag"
+ assert meta["failure_phase"] == "embedding"
+ assert meta["actionable_by"] == "USER_ACTIONABLE"
+ assert "API key" in meta["user_facing_message"]
+ assert "{" not in meta["user_facing_message"]
+
+ def test_langflow_disabled_api_key_failure(self, task_service):
+ """Disabled (not deleted) IBM keys must not fall through to generic Langflow copy."""
+ ft = _make_file_task(
+ phase=IngestionPhase.LANGFLOW,
+ docling_status=DoclingPhaseStatus.SUCCESS,
+ error=(
+ "Error building Component Embedding Model: Failed to initialize IBM "
+ "WatsonX embedding model: Attempt of authenticating connection to "
+ "service failed, please validate your credentials. Error: "
+ '{"errorCode":"BXNIM0420E","errorMessage":"Provided API key is disabled."}'
+ ),
+ )
+ meta = task_service._infer_failure_metadata(ft)
+ assert meta is not None
+ assert meta["component"] == "openrag"
+ assert meta["failure_phase"] == "embedding"
+ assert meta["actionable_by"] == "USER_ACTIONABLE"
+ assert "disabled" in meta["user_facing_message"].lower()
+ assert "unexpectedly" not in meta["user_facing_message"].lower()
+ assert "error building component" not in meta["user_facing_message"].lower()
+ assert "error running graph" not in meta["user_facing_message"].lower()
+ assert "{" not in meta["user_facing_message"]
+
+ def test_langflow_disconnect_with_api_key_text_prefers_credentials(self, task_service):
+ ft = _make_file_task(
+ phase=IngestionPhase.LANGFLOW,
+ docling_status=DoclingPhaseStatus.SUCCESS,
+ error=("Server disconnected without sending a response. Incorrect API key provided"),
+ )
+ meta = task_service._infer_failure_metadata(ft)
+ assert meta is not None
+ assert meta["failure_phase"] == "embedding"
+ assert "API key" in meta["user_facing_message"]
+ assert "connection was lost" not in meta["user_facing_message"].lower()
+
def test_duplicate_file_error(self, task_service):
ft = _make_file_task(
phase=IngestionPhase.DOCLING,