An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
import type { LangfuseSpan } from '@langfuse/tracing'
import { propagateAttributes, startActiveObservation } from '@langfuse/tracing'
import type { StreamTextOnErrorCallback } from 'ai'
import { consumeStream, convertToModelMessages, smoothStream } from 'ai'
import { researcher } from '@/lib/agents/researcher'
import { buildDraftVersions, currentDraftVersion } from '@/lib/draft/draft'
import {
createPublicErrorResponse,
serializePublicError
} from '@/lib/errors/public-error'
import {
isToolFailureError,
serializeToolFailure
} from '@/lib/errors/tool-error'
import { nextCitationLabelNumber } from '@/lib/utils/citation'
import { isTracingEnabled } from '@/lib/utils/telemetry'
import { loadChat } from '../actions/chat'
import { generateChatTitle } from '../agents/title-generator'
import {
getMaxAllowedTokens,
shouldTruncateMessages,
truncateMessages
} from '../utils/context-window'
import { getTextFromParts } from '../utils/message-utils'
import { perfLog, perfTime } from '../utils/perf-logging'
import { isUsageLogging, logUsage } from '../utils/usage-logging'
import { resolveAttachmentSizes } from './helpers/attachment-sizes'
import { buildAttachmentTokenEstimates } from './helpers/attachment-token-estimates'
import { capHistoricalAttachments } from './helpers/cap-historical-attachments'
import { capHistoricalPastedContent } from './helpers/cap-historical-pasted-content'
import { compactHistoricalMessages } from './helpers/compact-historical-messages'
import { convertDataPart } from './helpers/convert-data-part'
import { assignDataPartNonces } from './helpers/data-part-nonce'
import { dedupeAttachments } from './helpers/dedupe-attachments'
import { describeTurnInput } from './helpers/describe-turn-input'
import {
attachCurrentDraft,
compactDraftsInMessages
} from './helpers/draft-context'
import { hasAnswerAfterToolFailure } from './helpers/has-answer-after-tool-failure'
import { hasResponseContentPart } from './helpers/has-response-content'
import {
EMPTY_RESPONSE_STATUS_MESSAGE,
isEmptyResponse
} from './helpers/is-empty-response'
import { logAPICallErrorDiagnostics } from './helpers/log-api-call-error'
import { persistStreamResults } from './helpers/persist-stream-results'
import { prepareMessages } from './helpers/prepare-messages'
import {
buildStreamErrorSpanUpdate,
type StreamErrorPhase,
type StreamErrorStage
} from './helpers/stream-error-diagnostics'
import { stripSpecFromMessages } from './helpers/strip-spec-from-messages'
import { summarizeCarriedContext } from './helpers/summarize-carried-context'
import {
COLD_START_HISTORY_TOKEN_LIMIT,
trimColdStartHistory
} from './helpers/trim-cold-start-history'
import type { StreamContext } from './helpers/types'
import { BaseStreamConfig } from './types'
import { langfuseSpanProcessor } from '@/instrumentation'
// Constants
const DEFAULT_CHAT_TITLE = 'Untitled'
export async function createChatStreamResponse(
config: BaseStreamConfig
): Promise<Response> {
const {
message,
model,
chatId,
userId,
trigger,
messageId,
abortSignal,
isNewChat,
searchMode,
onZeroPartError
} = config
// Verify that chatId is provided
if (!chatId) {
return new Response('Chat ID is required', {
status: 400,
statusText: 'Bad Request'
})
}
// Skip loading chat for new chats optimization
let initialChat = null
if (!isNewChat) {
try {
const loadChatStart = performance.now()
initialChat = await loadChat(chatId, userId)
perfTime('loadChat completed', loadChatStart)
} catch (error) {
logAPICallErrorDiagnostics(error)
console.error('Initial chat load error:', error)
return createPublicErrorResponse(error, {
status: 500,
statusText: 'Internal Server Error'
})
}
// Authorization check: if chat exists, it must belong to the user
if (initialChat && initialChat.userId !== userId) {
return new Response('You are not allowed to access this chat', {
status: 403,
statusText: 'Forbidden'
})
}
} else {
perfLog('loadChat skipped for new chat')
}
const executeStream = async (rootSpan?: LangfuseSpan): Promise<Response> => {
// Real OTel trace ID, stored in message metadata so feedback scores can
// be attached to this trace later
const parentTraceId = rootSpan?.traceId
let hasStreamError = false
// A failing tool is the page or the search service failing, not the turn:
// the agent keeps going and the answer still lands. Tracked apart from the
// stream's own failures so it can only become one when nothing was answered.
let hasToolFailure = false
let hasEmptyResponse = false
let streamError: unknown
// The agent stream call is preparation until its first generation starts.
let streamErrorPhase: StreamErrorPhase = 'preparation'
let streamErrorStage: StreamErrorStage = 'prepare-messages'
// Sampled where the failure happens: the client can disconnect before the
// span is closed, and by then the signal no longer says whether the user
// cancelled this turn or the failure was the model's own.
let streamErrorWasCancelled = false
// Overall IO of the trace lives on the root observation. A regenerate turn
// carries no incoming message, so its input comes from the prepared history.
let rootInput = describeTurnInput(message?.parts)
let rootOutput: string | undefined
let carriedContext: Record<string, number> | undefined
let contextWindowTruncated = false
let coldStartHistoryTrimmed = false
const endTracing = async () => {
if (rootSpan) {
const failureUpdate = hasStreamError
? buildStreamErrorSpanUpdate(streamError, {
requestWasCancelled: streamErrorWasCancelled,
phase: streamErrorPhase,
stage: streamErrorStage
})
: hasEmptyResponse
? {
level: 'ERROR' as const,
statusMessage: EMPTY_RESPONSE_STATUS_MESSAGE
}
: null
// A turn that failed mid-answer or produced no answer text still has
// whatever was streamed before it. That is not the turn's answer, so
// it is not recorded as one.
const hasAnswer = !hasStreamError && !hasEmptyResponse
// A failure update carries its own metadata (#945), so the two are
// merged rather than spread as siblings: whichever came last would
// otherwise drop the other one entirely.
const failureMetadata =
failureUpdate && 'metadata' in failureUpdate
? failureUpdate.metadata
: undefined
const metadata = {
...(carriedContext !== undefined && { carriedContext }),
...(contextWindowTruncated && { contextWindowTruncated }),
...(coldStartHistoryTrimmed && { coldStartHistoryTrimmed }),
...failureMetadata
}
const update = {
...(rootInput !== undefined && { input: rootInput }),
...(hasAnswer && rootOutput !== undefined && { output: rootOutput }),
...failureUpdate,
...(Object.keys(metadata).length > 0 && { metadata })
}
if (Object.keys(update).length > 0) rootSpan.update(update)
rootSpan.end()
await langfuseSpanProcessor.forceFlush()
}
}
// Create stream context with trace ID
const context: StreamContext = {
chatId,
userId,
modelId: `${model.providerId}:${model.id}`,
messageId,
trigger,
initialChat,
abortSignal,
parentTraceId,
isNewChat
}
// Declare titlePromise in outer scope for onFinish access
let titlePromise: Promise<string> | undefined
try {
// Prepare messages for the model
const prepareStart = performance.now()
perfLog(
`prepareMessages - Invoked: trigger=${trigger}, isNewChat=${isNewChat}`
)
const messagesToModel = await prepareMessages(context, message)
perfTime('prepareMessages completed (stream)', prepareStart)
if (rootInput === undefined) {
rootInput = describeTurnInput(
messagesToModel.findLast(m => m.role === 'user')?.parts
)
}
// The chat's draft as of this turn, rebuilt from history
const currentDraft = currentDraftVersion(
buildDraftVersions(messagesToModel)
)
// Get the researcher agent with search mode
streamErrorStage = 'build-agent'
const researchAgent = researcher({
model: context.modelId,
modelConfig: model,
chatId,
searchMode,
citationLabelSeed: nextCitationLabelNumber(messagesToModel),
currentDraft
})
streamErrorStage = 'transform-messages'
const messagesWithNonces = assignDataPartNonces(messagesToModel)
const messagesWithoutSpec = compactDraftsInMessages(
stripSpecFromMessages(messagesWithNonces)
)
const messagesWithAttachmentSizes = await resolveAttachmentSizes(
messagesWithoutSpec,
userId
)
const coldStartHistory = trimColdStartHistory(
capHistoricalPastedContent(messagesWithAttachmentSizes),
{
limit:
COLD_START_HISTORY_TOKEN_LIMIT > 0
? Math.min(
COLD_START_HISTORY_TOKEN_LIMIT,
getMaxAllowedTokens(model)
)
: 0,
modelId: model.id
}
)
coldStartHistoryTrimmed = coldStartHistory.trimmedAtCurrentTurn
const messagesToConvert = dedupeAttachments(
capHistoricalAttachments(
compactHistoricalMessages(coldStartHistory.messages)
)
)
carriedContext = summarizeCarriedContext(messagesToConvert)
const attachmentTokenEstimates =
buildAttachmentTokenEstimates(messagesToConvert)
// Convert to model messages and apply context window management
streamErrorStage = 'convert-messages'
let modelMessages = await convertToModelMessages(
attachCurrentDraft(messagesToConvert, currentDraft),
{ convertDataPart }
)
streamErrorStage = 'truncate-messages'
if (
shouldTruncateMessages(modelMessages, model, attachmentTokenEstimates)
) {
// Truncation drops old messages, and the draft must survive it
if (currentDraft) {
modelMessages = await convertToModelMessages(
attachCurrentDraft(messagesToConvert, currentDraft, {
atLatest: true
}),
{ convertDataPart }
)
}
const maxTokens = getMaxAllowedTokens(model)
const originalCount = modelMessages.length
modelMessages = truncateMessages(
modelMessages,
maxTokens,
model.id,
attachmentTokenEstimates
)
contextWindowTruncated = true
if (process.env.NODE_ENV === 'development') {
console.log(
`Context window limit reached. Truncating from ${originalCount} to ${modelMessages.length} messages`
)
}
}
// Start title generation in parallel if it's a new chat
streamErrorStage = 'start-title-generation'
if (!initialChat && message) {
const userContent = getTextFromParts(message.parts)
titlePromise = generateChatTitle({
userMessageContent: userContent,
modelId: context.modelId,
abortSignal
}).catch(error => {
console.error('Error generating title:', error)
return DEFAULT_CHAT_TITLE
})
}
const llmStart = performance.now()
perfLog(
`researchAgent.stream - Start: model=${context.modelId}, searchMode=${searchMode}`
)
streamErrorStage = 'start-stream'
// AgentStreamParameters omits onError, but it reaches streamText. A tool
// failure is classified here too rather than assumed away: the same error
// is recoverable whichever handler it arrives at.
const result = await researchAgent.stream({
messages: modelMessages,
abortSignal,
onError: ({ error }) => {
if (isToolFailureError(error)) {
hasToolFailure = true
} else {
hasStreamError = true
}
streamError = error
streamErrorWasCancelled = abortSignal?.aborted ?? false
streamErrorPhase = 'generation'
},
experimental_transform: smoothStream({ chunking: 'word' }),
...(isUsageLogging() && {
onStepEnd: step => {
logUsage(
{ scope: 'step', modelId: context.modelId },
step.usage,
step.providerMetadata
)
}
})
} as Parameters<typeof researchAgent.stream>[0] & {
onError: StreamTextOnErrorCallback
})
result.consumeStream()
// Log the session-total usage once the stream settles (does not block the
// response; consumeStream above already drives it to completion).
if (isUsageLogging()) {
Promise.resolve(result.usage)
.then(usage =>
logUsage({ scope: 'total', modelId: context.modelId }, usage)
)
.catch(() => {})
}
return result.toUIMessageStreamResponse({
messageMetadata: ({ part }) => {
if (part.type === 'start') {
return {
traceId: parentTraceId,
searchMode,
modelId: context.modelId
}
}
},
onEnd: async ({ responseMessage, isAborted }) => {
try {
perfTime('researchAgent.stream completed', llmStart)
if (isAborted) return
if (!responseMessage) {
if (hasStreamError || hasToolFailure) {
hasStreamError = true
await onZeroPartError?.()
}
return
}
rootOutput = getTextFromParts(responseMessage.parts) || undefined
hasEmptyResponse = isEmptyResponse(responseMessage)
// The tool failure is only the turn's failure once the turn has no
// answer to show for it, and an answer counts only when it came
// after the failure: text the model wrote on its way to the tool is
// a preamble.
if (hasToolFailure && !hasAnswerAfterToolFailure(responseMessage)) {
hasStreamError = true
}
if (
hasEmptyResponse ||
(hasStreamError && !hasResponseContentPart(responseMessage))
) {
await onZeroPartError?.()
}
// Persist stream results to database
await persistStreamResults(
responseMessage,
chatId,
userId,
titlePromise,
parentTraceId,
searchMode,
context.modelId,
context.pendingInitialSave,
context.pendingInitialUserMessage
)
} finally {
await endTracing()
}
},
onError: (error: unknown) => {
streamError = error
streamErrorWasCancelled = abortSignal?.aborted ?? false
streamErrorPhase = 'generation'
if (isToolFailureError(error)) {
hasToolFailure = true
console.error('Tool failure:', error)
return serializeToolFailure(error)
}
hasStreamError = true
logAPICallErrorDiagnostics(error)
console.error('Stream response error:', error)
return serializePublicError(error)
},
consumeSseStream: consumeStream
})
} catch (error) {
hasStreamError = true
streamError = error
streamErrorWasCancelled = abortSignal?.aborted ?? false
await endTracing()
logAPICallErrorDiagnostics(error)
console.error('Stream execution error:', error)
return createPublicErrorResponse(error, {
status: 500,
statusText: 'Internal Server Error'
})
}
}
if (!isTracingEnabled()) {
return executeStream()
}
// Wrap execution in a root Langfuse observation so the researcher and
// title-generation spans share a single trace
return propagateAttributes(
{
traceName: 'research',
userId,
sessionId: chatId,
metadata: {
chatId,
userId,
modelId: `${model.providerId}:${model.id}`,
...(trigger && { trigger })
}
},
() =>
startActiveObservation('research', span => executeStream(span), {
endOnExit: false
})
)
}