Skip to content

An AI-powered search engine with a generative UI

6.4 KB
Raw Blame
import type { UIMessage } from 'ai'

import { estimateAttachmentTokens } from '@/lib/utils/attachment-tokens'

import { describeAttachment, isFilePart } from './attachment-parts'
import { isSourceContextMessage } from './compact-historical-messages'

const DEFAULT_REPLAY_LIMIT = 10
const DEFAULT_ATTACHMENT_TOKEN_BUDGET = 200_000
const MIN_WEIGHT_BLOCK_ATTACHMENTS = 2

/**
 * Only an explicit `0` disables the count cap. The weight budget below is a
 * separate knob and keeps applying.
 *
 * Templated environments routinely define a variable as an empty string, and
 * `Number('')` is `0` — which would silently replay every attachment while
 * looking configured. Anything unusable falls back to the default instead, and
 * a positive value always keeps at least one attachment.
 */
export function parseReplayLimit(raw: string | undefined): number {
  const value = raw?.trim()
  if (!value) return DEFAULT_REPLAY_LIMIT

  const parsed = Number(value)
  if (!Number.isFinite(parsed) || parsed < 0) return DEFAULT_REPLAY_LIMIT

  return parsed === 0 ? 0 : Math.max(1, Math.floor(parsed))
}

export const HISTORY_ATTACHMENT_REPLAY_LIMIT = parseReplayLimit(
  process.env.HISTORY_ATTACHMENT_REPLAY_LIMIT
)

export function parseAttachmentTokenBudget(raw: string | undefined): number {
  const value = raw?.trim()
  if (!value) return DEFAULT_ATTACHMENT_TOKEN_BUDGET

  const parsed = Number(value)
  if (!Number.isFinite(parsed) || parsed < 0) {
    return DEFAULT_ATTACHMENT_TOKEN_BUDGET
  }

  return parsed === 0 ? 0 : Math.max(1, Math.floor(parsed))
}

export const HISTORY_ATTACHMENT_TOKEN_BUDGET = parseAttachmentTokenBudget(
  process.env.HISTORY_ATTACHMENT_TOKEN_BUDGET
)

/**
 * Number of leading attachments to drop, quantized to whole blocks of `limit`.
 *
 * A plain "keep the newest N" boundary moves by one on every turn that adds an
 * attachment, and because the boundary sits near the START of the conversation
 * it would rewrite the prompt prefix — and so drop the provider's cache for the
 * whole request — on exactly the long threads this is meant to make cheaper.
 *
 * Quantizing means the dropped set is identical for every turn between two
 * block crossings, so the prefix survives; the cache is only reset once per
 * `limit` new attachments. The cost of that is a looser bound: the number kept
 * lands anywhere in [limit, 2 * limit).
 */
function getDroppedCount(total: number, limit: number): number {
  if (total <= limit) return 0
  return Math.floor((total - limit) / limit) * limit
}

/**
 * Number of leading attachments to drop, quantized by cumulative token weight.
 *
 * Blocks are anchored to the start of the surviving history and complete once
 * they reach `tokenBudget`, with at least two attachments per block. The
 * minimum is the stability bound: even an attachment heavier than the entire
 * budget cannot make the boundary advance on consecutive turns. We retain the
 * newest complete block plus any incomplete tail, so appending an attachment
 * cannot move the boundary until it completes another block.
 *
 * This deliberately permits nearly two blocks of replayed weight. A block can
 * exceed the budget by its final attachment, or more when an attachment alone
 * exceeds the budget. That overshoot is the cost of prefix stability while
 * guaranteeing that at least one historical attachment always survives.
 */
export function getWeightDroppedCount(
  weights: number[],
  tokenBudget: number,
  minBlockSize: number = MIN_WEIGHT_BLOCK_ATTACHMENTS
): number {
  let blockTokens = 0
  let blockAttachmentCount = 0
  let latestBlockEnd = 0
  let droppedCount = 0

  for (let i = 0; i < weights.length; i++) {
    blockTokens += weights[i]
    blockAttachmentCount += 1
    if (blockTokens < tokenBudget || blockAttachmentCount < minBlockSize) {
      continue
    }

    droppedCount = latestBlockEnd
    latestBlockEnd = i + 1
    blockTokens = 0
    blockAttachmentCount = 0
  }

  return droppedCount
}

/**
 * Caps how many of a conversation's attachments are replayed to the model.
 *
 * Attachments are re-sent on every turn, and an image costs thousands of input
 * tokens, so a thread that collects a few screenshots per turn ends up spending
 * most of its context re-reading pictures it already answered about. Older ones
 * become a text placeholder naming the file, which keeps the message valid when
 * it held nothing but attachments, and lets the model ask for a re-attach
 * instead of silently losing the reference. The file itself is untouched: it
 * stays stored and stays visible in the conversation.
 *
 * The newest user message is never capped — that is the request being answered.
 */
export function capHistoricalAttachments(
  messages: UIMessage[],
  limit: number = HISTORY_ATTACHMENT_REPLAY_LIMIT,
  tokenBudget: number = HISTORY_ATTACHMENT_TOKEN_BUDGET
): UIMessage[] {
  if (limit <= 0 && tokenBudget <= 0) return messages

  const currentTurnIndex = messages.findLastIndex(
    message => message.role === 'user' && !isSourceContextMessage(message)
  )
  const historyEnd =
    currentTurnIndex === -1 ? messages.length : currentTurnIndex

  let total = 0
  for (let i = 0; i < historyEnd; i++) {
    total += messages[i].parts.filter(isFilePart).length
  }

  const countDropCount = limit > 0 ? getDroppedCount(total, limit) : 0
  const survivingWeights: number[] = []
  let seenForBudget = 0

  for (let i = 0; i < historyEnd; i++) {
    for (const part of messages[i].parts) {
      if (!isFilePart(part)) continue

      seenForBudget += 1
      if (seenForBudget <= countDropCount) continue

      const file = part as {
        mediaType?: string
        size?: number
      }
      survivingWeights.push(estimateAttachmentTokens(file))
    }
  }

  const weightDropCount =
    tokenBudget > 0 ? getWeightDroppedCount(survivingWeights, tokenBudget) : 0

  const dropCount = countDropCount + weightDropCount
  if (dropCount === 0) return messages

  let seen = 0
  return messages.map((message, index) => {
    if (index >= historyEnd || !message.parts.some(isFilePart)) {
      return message
    }

    const parts = message.parts.map(part => {
      if (!isFilePart(part)) return part

      seen += 1
      if (seen > dropCount) return part

      const file = part as { mediaType?: string; filename?: string }
      return {
        type: 'text' as const,
        text: `[Attachment omitted from history: ${describeAttachment(
          file
        )}. Ask the user to re-attach it if you need to look at it again.]`
      }
    })

    return { ...message, parts }
  })
}