OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
refactor: Cleanup useChatStreaming logic (#1558)
Mike Fortman48fd2bfe8c26
diff --git a/frontend/hooks/useChatStreaming.ts b/frontend/hooks/useChatStreaming.tsindex c5f5b43c1..9a4e763f6 100644--- a/frontend/hooks/useChatStreaming.ts+++ b/frontend/hooks/useChatStreaming.ts@@ -5,6 +5,13 @@ import type {TokenUsage,} from "@/app/chat/_types/types";import { useChat } from "@/contexts/chat-context";+import {+ detectImplicitToolCall,+ detectRAGFromContent,+ parseOpenAIChatChunk,+ parseOpenRAGChunk,+ parseRealtimeChunk,+} from "@/lib/chat-stream-parsers";import type { FilterInput } from "@/lib/filter-normalization";import { buildSearchPayloadFilters } from "@/lib/filter-normalization";@@ -135,13 +142,12 @@ export function useChatStreaming({const decoder = new TextDecoder();let buffer = "";- let currentContent = "";+ const content = { value: "" };const currentFunctionCalls: FunctionCall[] = [];let newResponseId: string | null = null;let isError = false;- let usageData: TokenUsage | undefined;+ const usage: { value: TokenUsage | undefined } = { value: undefined };- // Initialize streaming messageif (!controller.signal.aborted && thisStreamId === streamIdRef.current) {setStreamingMessage({role: "assistant",@@ -173,394 +179,38 @@ export function useChatStreaming({try {const chunk = JSON.parse(line);- // Investigation logging for Granite 3.3 8b tool call detection- const chunkKeys = Object.keys(chunk);- const toolRelatedKeys = chunkKeys.filter(- (key) =>- key.toLowerCase().includes("tool") ||- key.toLowerCase().includes("call") ||- key.toLowerCase().includes("retrieval") ||- key.toLowerCase().includes("function") ||- key.toLowerCase().includes("result"),- );- if (toolRelatedKeys.length > 0) {- console.log(- "[Tool Detection] Found tool-related keys:",- toolRelatedKeys,- chunk,- );- }-- // Extract response ID if presentif (chunk.id) {newResponseId = chunk.id;} else if (chunk.response_id) {newResponseId = chunk.response_id;}- // Handle OpenAI Chat Completions streaming format- if (chunk.object === "response.chunk" && chunk.delta) {- // Handle function calls in delta- if (chunk.delta.function_call) {- if (chunk.delta.function_call.name) {- const functionCall: FunctionCall = {- name: chunk.delta.function_call.name,- arguments: undefined,- status: "pending",- argumentsString:- chunk.delta.function_call.arguments || "",- };- currentFunctionCalls.push(functionCall);- } else if (chunk.delta.function_call.arguments) {- const lastFunctionCall =- currentFunctionCalls[currentFunctionCalls.length - 1];- if (lastFunctionCall) {- if (!lastFunctionCall.argumentsString) {- lastFunctionCall.argumentsString = "";- }- lastFunctionCall.argumentsString +=- chunk.delta.function_call.arguments;-- if (lastFunctionCall.argumentsString.includes("}")) {- try {- const parsed = JSON.parse(- lastFunctionCall.argumentsString,- );- lastFunctionCall.arguments = parsed;- lastFunctionCall.status = "completed";- } catch (e) {- // Arguments not yet complete- }- }- }- }- }- // Handle tool calls in delta- else if (- chunk.delta.tool_calls &&- Array.isArray(chunk.delta.tool_calls)- ) {- for (const toolCall of chunk.delta.tool_calls) {- if (toolCall.function) {- if (toolCall.function.name) {- const functionCall: FunctionCall = {- name: toolCall.function.name,- arguments: undefined,- status: "pending",- argumentsString: toolCall.function.arguments || "",- };- currentFunctionCalls.push(functionCall);- } else if (toolCall.function.arguments) {- const lastFunctionCall =- currentFunctionCalls[- currentFunctionCalls.length - 1- ];- if (lastFunctionCall) {- if (!lastFunctionCall.argumentsString) {- lastFunctionCall.argumentsString = "";- }- lastFunctionCall.argumentsString +=- toolCall.function.arguments;-- if (- lastFunctionCall.argumentsString.includes("}")- ) {- try {- const parsed = JSON.parse(- lastFunctionCall.argumentsString,- );- lastFunctionCall.arguments = parsed;- lastFunctionCall.status = "completed";- } catch (e) {- // Arguments not yet complete- }- }- }- }- }- }- }- // Handle content/text in delta- else if (chunk.delta.content) {- currentContent += chunk.delta.content;- }-- // Handle finish reason- if (chunk.delta.finish_reason) {- currentFunctionCalls.forEach((fc) => {- if (fc.status === "pending" && fc.argumentsString) {- try {- fc.arguments = JSON.parse(fc.argumentsString);- fc.status = "completed";- } catch (e) {- fc.arguments = { raw: fc.argumentsString };- fc.status = "error";- }- }- });- }- }- // Handle Realtime API format - function call added- else if (- chunk.type === "response.output_item.added" &&- chunk.item?.type === "function_call"- ) {- let existing = currentFunctionCalls.find(- (fc) => fc.id === chunk.item.id,- );- if (!existing) {- existing = [...currentFunctionCalls]- .reverse()- .find(- (fc) =>- fc.status === "pending" &&- !fc.id &&- fc.name === (chunk.item.tool_name || chunk.item.name),- );- }-- if (existing) {- existing.id = chunk.item.id;- existing.type = chunk.item.type;- existing.name =- chunk.item.tool_name || chunk.item.name || existing.name;- existing.arguments =- chunk.item.inputs || existing.arguments;- } else {- const functionCall: FunctionCall = {- name:- chunk.item.tool_name || chunk.item.name || "unknown",- arguments: chunk.item.inputs || undefined,- status: "pending",- argumentsString: "",- id: chunk.item.id,- type: chunk.item.type,- };- currentFunctionCalls.push(functionCall);- }- }- // Handle Realtime API format - tool call added- else if (- chunk.type === "response.output_item.added" &&- chunk.item?.type?.includes("_call") &&- chunk.item?.type !== "function_call"- ) {- let existing = currentFunctionCalls.find(- (fc) => fc.id === chunk.item.id,- );- if (!existing) {- existing = [...currentFunctionCalls]- .reverse()- .find(- (fc) =>- fc.status === "pending" &&- !fc.id &&- fc.name ===- (chunk.item.tool_name ||- chunk.item.name ||- chunk.item.type),- );- }-- if (existing) {- existing.id = chunk.item.id;- existing.type = chunk.item.type;- existing.name =- chunk.item.tool_name ||- chunk.item.name ||- chunk.item.type ||- existing.name;- existing.arguments =- chunk.item.inputs || existing.arguments;- } else {- const functionCall = {- name:- chunk.item.tool_name ||- chunk.item.name ||- chunk.item.type ||- "unknown",- arguments: chunk.item.inputs || {},- status: "pending" as const,- id: chunk.item.id,- type: chunk.item.type,- };- currentFunctionCalls.push(functionCall);- }- }- // Handle function call done- else if (- chunk.type === "response.output_item.done" &&- chunk.item?.type === "function_call"- ) {- const functionCall = currentFunctionCalls.find(- (fc) =>- fc.id === chunk.item.id ||- fc.name === chunk.item.tool_name ||- fc.name === chunk.item.name,- );-- if (functionCall) {- functionCall.status =- chunk.item.status === "completed" ? "completed" : "error";- functionCall.id = chunk.item.id;- functionCall.type = chunk.item.type;- functionCall.name =- chunk.item.tool_name ||- chunk.item.name ||- functionCall.name;- functionCall.arguments =- chunk.item.inputs || functionCall.arguments;-- if (chunk.item.results) {- functionCall.result = chunk.item.results;- }- }- }- // Handle tool call done with results- else if (- chunk.type === "response.output_item.done" &&- chunk.item?.type?.includes("_call") &&- chunk.item?.type !== "function_call"- ) {- const functionCall = currentFunctionCalls.find(- (fc) =>- fc.id === chunk.item.id ||- fc.name === chunk.item.tool_name ||- fc.name === chunk.item.name ||- fc.name === chunk.item.type ||- fc.name.includes(chunk.item.type.replace("_call", "")) ||- chunk.item.type.includes(fc.name),- );-- if (functionCall) {- functionCall.arguments =- chunk.item.inputs || functionCall.arguments;- functionCall.status =- chunk.item.status === "completed" ? "completed" : "error";- functionCall.id = chunk.item.id;- functionCall.type = chunk.item.type;-- if (chunk.item.results) {- functionCall.result = chunk.item.results;- }- } else {- const newFunctionCall = {- name:- chunk.item.tool_name ||- chunk.item.name ||- chunk.item.type ||- "unknown",- arguments: chunk.item.inputs || {},- status: "completed" as const,- id: chunk.item.id,- type: chunk.item.type,- result: chunk.item.results,- };- currentFunctionCalls.push(newFunctionCall);- }- }-- // Handle text output streaming (Realtime API)- else if (chunk.type === "response.output_text.delta") {- currentContent += chunk.delta || "";- }- // Handle response.completed event - capture usage- else if (- chunk.type === "response.completed" &&- chunk.response?.usage- ) {- usageData = chunk.response.usage;- }- // Handle OpenRAG backend format- else if (chunk.output_text) {- currentContent += chunk.output_text;- }- // Note: chunk.delta.content is already handled in the OpenAI format section above (line 271)- // Only handle delta if it's a string or has text (not content)- else if (chunk.delta) {- if (typeof chunk.delta === "string") {- currentContent += chunk.delta;- } else if (- typeof chunk.delta === "object" &&- chunk.delta.text &&- !chunk.delta.content- ) {- // Only add text if content wasn't already processed- currentContent += chunk.delta.text;- }- }-- // Heuristic detection for implicit tool calls (Granite 3.3 8b workaround)- // Check if chunk contains retrieval results without explicit tool call markers- const hasImplicitToolCall =- // Check for various result indicators in the chunk- (chunk.results &&- Array.isArray(chunk.results) &&- chunk.results.length > 0) ||- (chunk.outputs &&- Array.isArray(chunk.outputs) &&- chunk.outputs.length > 0) ||- // Check for retrieval-related fields- chunk.retrieved_documents ||- chunk.retrieval_results ||- // Check for nested data structures that might contain results- (chunk.data &&- typeof chunk.data === "object" &&- (chunk.data.results ||- chunk.data.retrieved_documents ||- chunk.data.retrieval_results));-- if (hasImplicitToolCall && currentFunctionCalls.length === 0) {- console.log(- "[Heuristic Detection] Detected implicit tool call:",+ parseOpenAIChatChunk(chunk, content, currentFunctionCalls) ||+ parseRealtimeChunk(chunk,- );-- // Create a synthetic function call for the UI- const results =- chunk.results ||- chunk.outputs ||- chunk.retrieved_documents ||- chunk.retrieval_results ||- chunk.data?.results ||- chunk.data?.retrieved_documents ||- [];-- const syntheticFunctionCall: FunctionCall = {- name: "Retrieval",- arguments: { implicit: true, detected_heuristically: true },- status: "completed",- type: "retrieval_call",- result: results,- };- currentFunctionCalls.push(syntheticFunctionCall);- console.log(- "[Heuristic Detection] Created synthetic function call",- );- }+ content,+ currentFunctionCalls,+ usage,+ ) ||+ parseOpenRAGChunk(chunk, content);+ detectImplicitToolCall(chunk, currentFunctionCalls);- // Check for error status from Langflowif (chunk.finish_reason === "error" ||chunk.status === "failed") {console.error("Error detected in stream");-- // Mark this as an error message and complete the streamisError = true;-- // Exit the streaming loop by breaking the labeled while loopbreak streamLoop;}- // Update streaming message in real-timeif (!controller.signal.aborted &&thisStreamId === streamIdRef.current) {setStreamingMessage({role: "assistant",- content: currentContent,+ content: content.value,functionCalls:currentFunctionCalls.length > 0? [...currentFunctionCalls]@@ -580,57 +230,29 @@ export function useChatStreaming({if (timeoutId) clearTimeout(timeoutId);}- // Check if we got any content at allif (!hasReceivedData ||- (!currentContent && currentFunctionCalls.length === 0)+ (!content.value && currentFunctionCalls.length === 0)) {throw new Error("No response received from the server. Please try again.",);}- // Post-processing: Heuristic detection based on final content- // If no explicit tool calls detected but content shows RAG indicators- if (currentFunctionCalls.length === 0 && currentContent) {- // Check for citation patterns that indicate RAG usage- const hasCitations =- /\(Source:|\[Source:|\bSource:|filename:|document:/i.test(- currentContent,- );- // Check for common RAG response patterns- const hasRAGPattern =- /based on.*(?:document|file|information|data)|according to.*(?:document|file)/i.test(- currentContent,- );-- if (hasCitations || hasRAGPattern) {- console.log(- "[Post-Processing] Detected RAG usage from content patterns",- );- const syntheticFunctionCall: FunctionCall = {- name: "Retrieval",- arguments: {- implicit: true,- detected_from: hasCitations ? "citations" : "content_patterns",- },- status: "completed",- type: "retrieval_call",- };- currentFunctionCalls.push(syntheticFunctionCall);- }+ if (currentFunctionCalls.length === 0 && content.value) {+ const ragCall = detectRAGFromContent(content.value);+ if (ragCall) currentFunctionCalls.push(ragCall);}- // Finalize the messageconst finalMessage: Message = {role: "assistant",- content: currentContent,+ content: content.value,functionCalls:currentFunctionCalls.length > 0 ? currentFunctionCalls : undefined,timestamp: new Date(),isStreaming: false,- error: isError, // Mark as error if Langflow returned error status- usage: usageData,+ error: isError,+ usage: usage.value,};if (!controller.signal.aborted && thisStreamId === streamIdRef.current) {diff --git a/frontend/lib/chat-stream-parsers.ts b/frontend/lib/chat-stream-parsers.tsnew file mode 100644index 000000000..61d78fa80--- /dev/null+++ b/frontend/lib/chat-stream-parsers.ts@@ -0,0 +1,361 @@+import type { FunctionCall, TokenUsage } from "@/app/chat/_types/types";++type Chunk = Record<string, unknown>;++export function parseOpenAIChatChunk(+ chunk: unknown,+ content: { value: string },+ calls: FunctionCall[],+): boolean {+ const c = chunk as Chunk;+ if (!(c.object === "response.chunk" && c.delta)) return false;++ const delta = c.delta as Chunk;++ if (delta.function_call) {+ const fc = delta.function_call as Chunk;+ if (fc.name) {+ calls.push({+ name: fc.name as string,+ arguments: undefined,+ status: "pending",+ argumentsString: (fc.arguments as string) || "",+ });+ } else if (fc.arguments) {+ const last = calls[calls.length - 1];+ if (last) {+ last.argumentsString =+ (last.argumentsString ?? "") + (fc.arguments as string);+ if (last.argumentsString.includes("}")) {+ try {+ last.arguments = JSON.parse(last.argumentsString);+ last.status = "completed";+ } catch {+ // arguments not yet complete+ }+ }+ }+ }+ } else if (delta.tool_calls && Array.isArray(delta.tool_calls)) {+ for (const toolCall of delta.tool_calls as Chunk[]) {+ const fn = toolCall.function as Chunk | undefined;+ if (!fn) continue;+ if (fn.name) {+ calls.push({+ name: fn.name as string,+ arguments: undefined,+ status: "pending",+ argumentsString: (fn.arguments as string) || "",+ });+ } else if (fn.arguments) {+ const idx =+ typeof toolCall.index === "number"+ ? (toolCall.index as number)+ : undefined;+ let target: FunctionCall | undefined;+ if (idx === undefined) {+ target = calls[calls.length - 1];+ } else if (idx >= 0 && idx < calls.length) {+ target = calls[idx];+ }+ if (target) {+ target.argumentsString =+ (target.argumentsString ?? "") + (fn.arguments as string);+ if (target.argumentsString.includes("}")) {+ try {+ target.arguments = JSON.parse(target.argumentsString);+ target.status = "completed";+ } catch {+ // arguments not yet complete+ }+ }+ }+ }+ }+ } else if (delta.content) {+ content.value += delta.content as string;+ }++ if (delta.finish_reason) {+ for (const fc of calls) {+ if (fc.status === "pending" && fc.argumentsString) {+ try {+ fc.arguments = JSON.parse(fc.argumentsString);+ fc.status = "completed";+ } catch {+ fc.arguments = { raw: fc.argumentsString };+ fc.status = "error";+ }+ }+ }+ }++ return true;+}++export function parseRealtimeChunk(+ chunk: unknown,+ content: { value: string },+ calls: FunctionCall[],+ usage: { value: TokenUsage | undefined },+): boolean {+ const c = chunk as Chunk;+ const type = c.type as string | undefined;+ if (!type) return false;++ const item = c.item as Chunk | undefined;++ if (type === "response.output_item.added" && item?.type === "function_call") {+ let existing = calls.find((fc) => fc.id === item.id);+ if (!existing) {+ existing = [...calls]+ .reverse()+ .find(+ (fc) =>+ fc.status === "pending" &&+ !fc.id &&+ fc.name === (item.tool_name || item.name),+ );+ }+ if (existing) {+ existing.id = item.id as string;+ existing.type = item.type as string;+ existing.name = (item.tool_name || item.name || existing.name) as string;+ existing.arguments = (item.inputs || existing.arguments) as Record<+ string,+ unknown+ >;+ } else {+ calls.push({+ name: (item.tool_name || item.name || "unknown") as string,+ arguments: item.inputs as Record<string, unknown> | undefined,+ status: "pending",+ argumentsString: "",+ id: item.id as string,+ type: item.type as string,+ });+ }+ return true;+ }++ if (+ type === "response.output_item.added" &&+ typeof item?.type === "string" &&+ item.type.includes("_call") &&+ item.type !== "function_call"+ ) {+ let existing = calls.find((fc) => fc.id === item.id);+ if (!existing) {+ existing = [...calls]+ .reverse()+ .find(+ (fc) =>+ fc.status === "pending" &&+ !fc.id &&+ fc.name === (item.tool_name || item.name || item.type),+ );+ }+ if (existing) {+ existing.id = item.id as string;+ existing.type = item.type as string;+ existing.name = (item.tool_name ||+ item.name ||+ item.type ||+ existing.name) as string;+ existing.arguments = (item.inputs || existing.arguments) as Record<+ string,+ unknown+ >;+ } else {+ calls.push({+ name: (item.tool_name || item.name || item.type || "unknown") as string,+ arguments: (item.inputs || {}) as Record<string, unknown>,+ status: "pending",+ id: item.id as string,+ type: item.type as string,+ });+ }+ return true;+ }++ if (type === "response.output_item.done" && item?.type === "function_call") {+ const functionCall = calls.find(+ (fc) =>+ fc.id === item.id ||+ fc.name === item.tool_name ||+ fc.name === item.name,+ );+ if (functionCall) {+ functionCall.status = item.status === "completed" ? "completed" : "error";+ functionCall.id = item.id as string;+ functionCall.type = item.type as string;+ functionCall.name = (item.tool_name ||+ item.name ||+ functionCall.name) as string;+ functionCall.arguments = (item.inputs ||+ functionCall.arguments) as Record<string, unknown>;+ if (item.results)+ functionCall.result = item.results as FunctionCall["result"];+ }+ return true;+ }++ if (+ type === "response.output_item.done" &&+ typeof item?.type === "string" &&+ item.type.includes("_call") &&+ item.type !== "function_call"+ ) {+ const functionCall = calls.find(+ (fc) =>+ fc.id === item.id ||+ fc.name === item.tool_name ||+ fc.name === item.name ||+ fc.name === item.type ||+ fc.name.includes((item.type as string).replace("_call", "")) ||+ (item.type as string).includes(fc.name),+ );+ if (functionCall) {+ functionCall.arguments = (item.inputs ||+ functionCall.arguments) as Record<string, unknown>;+ functionCall.status = item.status === "completed" ? "completed" : "error";+ functionCall.id = item.id as string;+ functionCall.type = item.type as string;+ if (item.results)+ functionCall.result = item.results as FunctionCall["result"];+ } else {+ calls.push({+ name: (item.tool_name || item.name || item.type || "unknown") as string,+ arguments: (item.inputs || {}) as Record<string, unknown>,+ status: "completed",+ id: item.id as string,+ type: item.type as string,+ result: item.results as FunctionCall["result"],+ });+ }+ return true;+ }++ if (type === "response.output_text.delta") {+ content.value += (c.delta as string) || "";+ return true;+ }++ if (type === "response.completed" && (c.response as Chunk)?.usage) {+ usage.value = (c.response as Chunk).usage as TokenUsage;+ return true;+ }++ return false;+}++export function parseOpenRAGChunk(+ chunk: unknown,+ content: { value: string },+): boolean {+ const c = chunk as Chunk;++ if (c.output_text) {+ content.value += c.output_text as string;+ return true;+ }++ if (c.delta) {+ if (typeof c.delta === "string") {+ content.value += c.delta;+ return true;+ }+ const delta = c.delta as Chunk;+ if (delta.text && !delta.content) {+ content.value += delta.text as string;+ return true;+ }+ }++ return false;+}++// Granite 3.3 8b workaround: detects implicit tool calls with no explicit tool call markers+export function detectImplicitToolCall(+ chunk: unknown,+ calls: FunctionCall[],+): void {+ if (calls.length > 0) return;++ const c = chunk as Chunk;++ const toolRelatedKeys = Object.keys(c).filter(+ (key) =>+ key.toLowerCase().includes("tool") ||+ key.toLowerCase().includes("call") ||+ key.toLowerCase().includes("retrieval") ||+ key.toLowerCase().includes("function") ||+ key.toLowerCase().includes("result"),+ );+ if (toolRelatedKeys.length > 0) {+ console.log(+ "[Tool Detection] Found tool-related keys:",+ toolRelatedKeys,+ chunk,+ );+ }++ const data = c.data as Chunk | undefined;++ const nonEmpty = (v: unknown): v is unknown[] =>+ Array.isArray(v) && (v as unknown[]).length > 0;++ const hasImplicitToolCall =+ nonEmpty(c.results) ||+ nonEmpty(c.outputs) ||+ nonEmpty(c.retrieved_documents) ||+ nonEmpty(c.retrieval_results) ||+ (data &&+ (nonEmpty(data.results) ||+ nonEmpty(data.retrieved_documents) ||+ nonEmpty(data.retrieval_results)));++ if (!hasImplicitToolCall) return;++ console.log("[Heuristic Detection] Detected implicit tool call:", chunk);+ const results =+ (nonEmpty(c.results) && c.results) ||+ (nonEmpty(c.outputs) && c.outputs) ||+ (nonEmpty(c.retrieved_documents) && c.retrieved_documents) ||+ (nonEmpty(c.retrieval_results) && c.retrieval_results) ||+ (nonEmpty(data?.results) && data?.results) ||+ (nonEmpty(data?.retrieved_documents) && data?.retrieved_documents) ||+ [];++ calls.push({+ name: "Retrieval",+ arguments: { implicit: true, detected_heuristically: true },+ status: "completed",+ type: "retrieval_call",+ result: results as FunctionCall["result"],+ });+ console.log("[Heuristic Detection] Created synthetic function call");+}++// Post-processing: detects RAG usage from citation/content patterns in final response text+export function detectRAGFromContent(content: string): FunctionCall | null {+ const hasCitations =+ /\(Source:|\[Source:|\bSource:|filename:|document:/i.test(content);+ const hasRAGPattern =+ /based on.*(?:document|file|information|data)|according to.*(?:document|file)/i.test(+ content,+ );++ if (!hasCitations && !hasRAGPattern) return null;++ console.log("[Post-Processing] Detected RAG usage from content patterns");+ return {+ name: "Retrieval",+ arguments: {+ implicit: true,+ detected_from: hasCitations ? "citations" : "content_patterns",+ },+ status: "completed",+ type: "retrieval_call",+ };+}