Skip to content

An AI-powered search engine with a generative UI

16.4 KB
Raw Blame
import type { LangfuseSpan } from '@langfuse/tracing'
import { propagateAttributes, startActiveObservation } from '@langfuse/tracing'
import type { StreamTextOnErrorCallback } from 'ai'
import { consumeStream, convertToModelMessages, smoothStream } from 'ai'

import { researcher } from '@/lib/agents/researcher'
import { buildDraftVersions, currentDraftVersion } from '@/lib/draft/draft'
import {
  createPublicErrorResponse,
  serializePublicError
} from '@/lib/errors/public-error'
import {
  isToolFailureError,
  serializeToolFailure
} from '@/lib/errors/tool-error'
import { nextCitationLabelNumber } from '@/lib/utils/citation'
import { isTracingEnabled } from '@/lib/utils/telemetry'

import { loadChat } from '../actions/chat'
import { generateChatTitle } from '../agents/title-generator'
import {
  getMaxAllowedTokens,
  shouldTruncateMessages,
  truncateMessages
} from '../utils/context-window'
import { getTextFromParts } from '../utils/message-utils'
import { perfLog, perfTime } from '../utils/perf-logging'
import { isUsageLogging, logUsage } from '../utils/usage-logging'

import { resolveAttachmentSizes } from './helpers/attachment-sizes'
import { buildAttachmentTokenEstimates } from './helpers/attachment-token-estimates'
import { capHistoricalAttachments } from './helpers/cap-historical-attachments'
import { capHistoricalPastedContent } from './helpers/cap-historical-pasted-content'
import { compactHistoricalMessages } from './helpers/compact-historical-messages'
import { convertDataPart } from './helpers/convert-data-part'
import { assignDataPartNonces } from './helpers/data-part-nonce'
import { dedupeAttachments } from './helpers/dedupe-attachments'
import { describeTurnInput } from './helpers/describe-turn-input'
import {
  attachCurrentDraft,
  compactDraftsInMessages
} from './helpers/draft-context'
import { hasAnswerAfterToolFailure } from './helpers/has-answer-after-tool-failure'
import { hasResponseContentPart } from './helpers/has-response-content'
import {
  EMPTY_RESPONSE_STATUS_MESSAGE,
  isEmptyResponse
} from './helpers/is-empty-response'
import { logAPICallErrorDiagnostics } from './helpers/log-api-call-error'
import { persistStreamResults } from './helpers/persist-stream-results'
import { prepareMessages } from './helpers/prepare-messages'
import {
  buildStreamErrorSpanUpdate,
  type StreamErrorPhase,
  type StreamErrorStage
} from './helpers/stream-error-diagnostics'
import { stripSpecFromMessages } from './helpers/strip-spec-from-messages'
import { summarizeCarriedContext } from './helpers/summarize-carried-context'
import {
  COLD_START_HISTORY_TOKEN_LIMIT,
  trimColdStartHistory
} from './helpers/trim-cold-start-history'
import type { StreamContext } from './helpers/types'
import { BaseStreamConfig } from './types'

import { langfuseSpanProcessor } from '@/instrumentation'

// Constants
const DEFAULT_CHAT_TITLE = 'Untitled'

export async function createChatStreamResponse(
  config: BaseStreamConfig
): Promise<Response> {
  const {
    message,
    model,
    chatId,
    userId,
    trigger,
    messageId,
    abortSignal,
    isNewChat,
    searchMode,
    onZeroPartError
  } = config

  // Verify that chatId is provided
  if (!chatId) {
    return new Response('Chat ID is required', {
      status: 400,
      statusText: 'Bad Request'
    })
  }

  // Skip loading chat for new chats optimization
  let initialChat = null
  if (!isNewChat) {
    try {
      const loadChatStart = performance.now()
      initialChat = await loadChat(chatId, userId)
      perfTime('loadChat completed', loadChatStart)
    } catch (error) {
      logAPICallErrorDiagnostics(error)
      console.error('Initial chat load error:', error)
      return createPublicErrorResponse(error, {
        status: 500,
        statusText: 'Internal Server Error'
      })
    }

    // Authorization check: if chat exists, it must belong to the user
    if (initialChat && initialChat.userId !== userId) {
      return new Response('You are not allowed to access this chat', {
        status: 403,
        statusText: 'Forbidden'
      })
    }
  } else {
    perfLog('loadChat skipped for new chat')
  }

  const executeStream = async (rootSpan?: LangfuseSpan): Promise<Response> => {
    // Real OTel trace ID, stored in message metadata so feedback scores can
    // be attached to this trace later
    const parentTraceId = rootSpan?.traceId
    let hasStreamError = false
    // A failing tool is the page or the search service failing, not the turn:
    // the agent keeps going and the answer still lands. Tracked apart from the
    // stream's own failures so it can only become one when nothing was answered.
    let hasToolFailure = false
    let hasEmptyResponse = false
    let streamError: unknown
    // The agent stream call is preparation until its first generation starts.
    let streamErrorPhase: StreamErrorPhase = 'preparation'
    let streamErrorStage: StreamErrorStage = 'prepare-messages'
    // Sampled where the failure happens: the client can disconnect before the
    // span is closed, and by then the signal no longer says whether the user
    // cancelled this turn or the failure was the model's own.
    let streamErrorWasCancelled = false
    // Overall IO of the trace lives on the root observation. A regenerate turn
    // carries no incoming message, so its input comes from the prepared history.
    let rootInput = describeTurnInput(message?.parts)
    let rootOutput: string | undefined
    let carriedContext: Record<string, number> | undefined
    let contextWindowTruncated = false
    let coldStartHistoryTrimmed = false

    const endTracing = async () => {
      if (rootSpan) {
        const failureUpdate = hasStreamError
          ? buildStreamErrorSpanUpdate(streamError, {
              requestWasCancelled: streamErrorWasCancelled,
              phase: streamErrorPhase,
              stage: streamErrorStage
            })
          : hasEmptyResponse
            ? {
                level: 'ERROR' as const,
                statusMessage: EMPTY_RESPONSE_STATUS_MESSAGE
              }
            : null
        // A turn that failed mid-answer or produced no answer text still has
        // whatever was streamed before it. That is not the turn's answer, so
        // it is not recorded as one.
        const hasAnswer = !hasStreamError && !hasEmptyResponse
        // A failure update carries its own metadata (#945), so the two are
        // merged rather than spread as siblings: whichever came last would
        // otherwise drop the other one entirely.
        const failureMetadata =
          failureUpdate && 'metadata' in failureUpdate
            ? failureUpdate.metadata
            : undefined
        const metadata = {
          ...(carriedContext !== undefined && { carriedContext }),
          ...(contextWindowTruncated && { contextWindowTruncated }),
          ...(coldStartHistoryTrimmed && { coldStartHistoryTrimmed }),
          ...failureMetadata
        }
        const update = {
          ...(rootInput !== undefined && { input: rootInput }),
          ...(hasAnswer && rootOutput !== undefined && { output: rootOutput }),
          ...failureUpdate,
          ...(Object.keys(metadata).length > 0 && { metadata })
        }
        if (Object.keys(update).length > 0) rootSpan.update(update)
        rootSpan.end()
        await langfuseSpanProcessor.forceFlush()
      }
    }

    // Create stream context with trace ID
    const context: StreamContext = {
      chatId,
      userId,
      modelId: `${model.providerId}:${model.id}`,
      messageId,
      trigger,
      initialChat,
      abortSignal,
      parentTraceId,
      isNewChat
    }

    // Declare titlePromise in outer scope for onFinish access
    let titlePromise: Promise<string> | undefined

    try {
      // Prepare messages for the model
      const prepareStart = performance.now()
      perfLog(
        `prepareMessages - Invoked: trigger=${trigger}, isNewChat=${isNewChat}`
      )
      const messagesToModel = await prepareMessages(context, message)
      perfTime('prepareMessages completed (stream)', prepareStart)

      if (rootInput === undefined) {
        rootInput = describeTurnInput(
          messagesToModel.findLast(m => m.role === 'user')?.parts
        )
      }

      // The chat's draft as of this turn, rebuilt from history
      const currentDraft = currentDraftVersion(
        buildDraftVersions(messagesToModel)
      )

      // Get the researcher agent with search mode
      streamErrorStage = 'build-agent'
      const researchAgent = researcher({
        model: context.modelId,
        modelConfig: model,
        chatId,
        searchMode,
        citationLabelSeed: nextCitationLabelNumber(messagesToModel),
        currentDraft
      })

      streamErrorStage = 'transform-messages'
      const messagesWithNonces = assignDataPartNonces(messagesToModel)
      const messagesWithoutSpec = compactDraftsInMessages(
        stripSpecFromMessages(messagesWithNonces)
      )
      const messagesWithAttachmentSizes = await resolveAttachmentSizes(
        messagesWithoutSpec,
        userId
      )
      const coldStartHistory = trimColdStartHistory(
        capHistoricalPastedContent(messagesWithAttachmentSizes),
        {
          limit:
            COLD_START_HISTORY_TOKEN_LIMIT > 0
              ? Math.min(
                  COLD_START_HISTORY_TOKEN_LIMIT,
                  getMaxAllowedTokens(model)
                )
              : 0,
          modelId: model.id
        }
      )
      coldStartHistoryTrimmed = coldStartHistory.trimmedAtCurrentTurn
      const messagesToConvert = dedupeAttachments(
        capHistoricalAttachments(
          compactHistoricalMessages(coldStartHistory.messages)
        )
      )
      carriedContext = summarizeCarriedContext(messagesToConvert)
      const attachmentTokenEstimates =
        buildAttachmentTokenEstimates(messagesToConvert)

      // Convert to model messages and apply context window management
      streamErrorStage = 'convert-messages'
      let modelMessages = await convertToModelMessages(
        attachCurrentDraft(messagesToConvert, currentDraft),
        { convertDataPart }
      )

      streamErrorStage = 'truncate-messages'
      if (
        shouldTruncateMessages(modelMessages, model, attachmentTokenEstimates)
      ) {
        // Truncation drops old messages, and the draft must survive it
        if (currentDraft) {
          modelMessages = await convertToModelMessages(
            attachCurrentDraft(messagesToConvert, currentDraft, {
              atLatest: true
            }),
            { convertDataPart }
          )
        }

        const maxTokens = getMaxAllowedTokens(model)
        const originalCount = modelMessages.length
        modelMessages = truncateMessages(
          modelMessages,
          maxTokens,
          model.id,
          attachmentTokenEstimates
        )
        contextWindowTruncated = true

        if (process.env.NODE_ENV === 'development') {
          console.log(
            `Context window limit reached. Truncating from ${originalCount} to ${modelMessages.length} messages`
          )
        }
      }

      // Start title generation in parallel if it's a new chat
      streamErrorStage = 'start-title-generation'
      if (!initialChat && message) {
        const userContent = getTextFromParts(message.parts)
        titlePromise = generateChatTitle({
          userMessageContent: userContent,
          modelId: context.modelId,
          abortSignal
        }).catch(error => {
          console.error('Error generating title:', error)
          return DEFAULT_CHAT_TITLE
        })
      }

      const llmStart = performance.now()
      perfLog(
        `researchAgent.stream - Start: model=${context.modelId}, searchMode=${searchMode}`
      )
      streamErrorStage = 'start-stream'
      // AgentStreamParameters omits onError, but it reaches streamText. A tool
      // failure is classified here too rather than assumed away: the same error
      // is recoverable whichever handler it arrives at.
      const result = await researchAgent.stream({
        messages: modelMessages,
        abortSignal,
        onError: ({ error }) => {
          if (isToolFailureError(error)) {
            hasToolFailure = true
          } else {
            hasStreamError = true
          }
          streamError = error
          streamErrorWasCancelled = abortSignal?.aborted ?? false
          streamErrorPhase = 'generation'
        },
        experimental_transform: smoothStream({ chunking: 'word' }),
        ...(isUsageLogging() && {
          onStepEnd: step => {
            logUsage(
              { scope: 'step', modelId: context.modelId },
              step.usage,
              step.providerMetadata
            )
          }
        })
      } as Parameters<typeof researchAgent.stream>[0] & {
        onError: StreamTextOnErrorCallback
      })
      result.consumeStream()

      // Log the session-total usage once the stream settles (does not block the
      // response; consumeStream above already drives it to completion).
      if (isUsageLogging()) {
        Promise.resolve(result.usage)
          .then(usage =>
            logUsage({ scope: 'total', modelId: context.modelId }, usage)
          )
          .catch(() => {})
      }

      return result.toUIMessageStreamResponse({
        messageMetadata: ({ part }) => {
          if (part.type === 'start') {
            return {
              traceId: parentTraceId,
              searchMode,
              modelId: context.modelId
            }
          }
        },
        onEnd: async ({ responseMessage, isAborted }) => {
          try {
            perfTime('researchAgent.stream completed', llmStart)
            if (isAborted) return
            if (!responseMessage) {
              if (hasStreamError || hasToolFailure) {
                hasStreamError = true
                await onZeroPartError?.()
              }
              return
            }

            rootOutput = getTextFromParts(responseMessage.parts) || undefined
            hasEmptyResponse = isEmptyResponse(responseMessage)
            // The tool failure is only the turn's failure once the turn has no
            // answer to show for it, and an answer counts only when it came
            // after the failure: text the model wrote on its way to the tool is
            // a preamble.
            if (hasToolFailure && !hasAnswerAfterToolFailure(responseMessage)) {
              hasStreamError = true
            }
            if (
              hasEmptyResponse ||
              (hasStreamError && !hasResponseContentPart(responseMessage))
            ) {
              await onZeroPartError?.()
            }

            // Persist stream results to database
            await persistStreamResults(
              responseMessage,
              chatId,
              userId,
              titlePromise,
              parentTraceId,
              searchMode,
              context.modelId,
              context.pendingInitialSave,
              context.pendingInitialUserMessage
            )
          } finally {
            await endTracing()
          }
        },
        onError: (error: unknown) => {
          streamError = error
          streamErrorWasCancelled = abortSignal?.aborted ?? false
          streamErrorPhase = 'generation'

          if (isToolFailureError(error)) {
            hasToolFailure = true
            console.error('Tool failure:', error)
            return serializeToolFailure(error)
          }

          hasStreamError = true
          logAPICallErrorDiagnostics(error)
          console.error('Stream response error:', error)
          return serializePublicError(error)
        },
        consumeSseStream: consumeStream
      })
    } catch (error) {
      hasStreamError = true
      streamError = error
      streamErrorWasCancelled = abortSignal?.aborted ?? false
      await endTracing()
      logAPICallErrorDiagnostics(error)
      console.error('Stream execution error:', error)
      return createPublicErrorResponse(error, {
        status: 500,
        statusText: 'Internal Server Error'
      })
    }
  }

  if (!isTracingEnabled()) {
    return executeStream()
  }

  // Wrap execution in a root Langfuse observation so the researcher and
  // title-generation spans share a single trace
  return propagateAttributes(
    {
      traceName: 'research',
      userId,
      sessionId: chatId,
      metadata: {
        chatId,
        userId,
        modelId: `${model.providerId}:${model.id}`,
        ...(trigger && { trigger })
      }
    },
    () =>
      startActiveObservation('research', span => executeStream(span), {
        endOnExit: false
      })
  )
}