Skip to content

An AI-powered search engine with a generative UI

fix(attachments): deduplicate repeated attachments instead of replaying every copy (#947)

Yoshiki Miura346c30c453a1
diff --git a/app/api/upload/__tests__/route.test.ts b/app/api/upload/__tests__/route.test.ts
new file mode 100644
index 00000000..776e4064
--- /dev/null
+++ b/app/api/upload/__tests__/route.test.ts
@@ -0,0 +1,193 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@/lib/auth/get-current-user', () => ({
+ getCurrentUserId: vi.fn(() => Promise.resolve('user-1'))
+}))
+
+vi.mock('@/lib/analytics/dispatch', () => ({
+ capture: vi.fn(() => Promise.resolve())
+}))
+
+vi.mock('@/lib/db/actions', () => ({
+ findChatFileCandidates: vi.fn(),
+ createLibraryFile: vi.fn()
+}))
+
+const send = vi.fn(() => Promise.resolve({}))
+
+vi.mock('@/lib/storage/r2-client', () => ({
+ getR2Client: vi.fn(() => ({ send })),
+ getSignedFileUrl: vi.fn((key: string) =>
+ Promise.resolve(`https://uploads.example.com/${key}?sig=abc`)
+ ),
+ isObjectStorageConfigured: vi.fn(() => true),
+ getChatFileObjectKeyPrefix: (userId: string, chatId: string) =>
+ `${userId}/chats/${chatId}/`,
+ getObjectContentMd5: vi.fn(() => Promise.resolve<string | null>(null)),
+ R2_BUCKET_NAME: 'test-bucket'
+}))
+
+import { createHash } from 'node:crypto'
+
+import * as dbActions from '@/lib/db/actions'
+import { getObjectContentMd5 } from '@/lib/storage/r2-client'
+
+import { POST } from '../route'
+
+const EXISTING = {
+ id: 'file-1',
+ userId: 'user-1',
+ chatId: 'chat-1',
+ filename: 'report.pdf',
+ objectKey: 'user-1/chats/chat-1/1700000000000-report.pdf',
+ mediaType: 'application/pdf',
+ size: 1234,
+ createdAt: new Date('2026-01-01'),
+ updatedAt: new Date('2026-01-01')
+}
+
+/**
+ * The route only reads headers and formData, so a stub avoids round-tripping a
+ * File through multipart encoding in the test environment.
+ */
+function uploadRequest(bytes = 1234) {
+ const file = {
+ name: 'report.pdf',
+ type: 'application/pdf',
+ size: bytes,
+ arrayBuffer: async () => new Uint8Array(bytes).buffer
+ }
+ const formData = new Map<string, unknown>([
+ ['file', file],
+ ['chatId', 'chat-1']
+ ])
+
+ return {
+ headers: { get: () => 'multipart/form-data; boundary=x' },
+ formData: async () => ({ get: (key: string) => formData.get(key) ?? null })
+ } as any
+}
+
+function md5OfUpload(bytes = 1234) {
+ return createHash('md5').update(Buffer.alloc(bytes)).digest('hex')
+}
+
+describe('POST /api/upload', () => {
+ beforeEach(() => {
+ vi.clearAllMocks()
+ delete process.env.ENABLE_AUTH
+ vi.mocked(getObjectContentMd5).mockResolvedValue(md5OfUpload())
+ vi.mocked(dbActions.createLibraryFile).mockResolvedValue({
+ ...EXISTING,
+ id: 'file-2',
+ objectKey: 'user-1/chats/chat-1/1800000000000-report.pdf'
+ } as any)
+ })
+
+ it('reuses the stored copy when the same file is uploaded again', async () => {
+ vi.mocked(dbActions.findChatFileCandidates).mockResolvedValue([
+ EXISTING
+ ] as any)
+
+ const { file } = await (await POST(uploadRequest())).json()
+
+ expect(file.key).toBe(EXISTING.objectKey)
+ expect(file.id).toBe(EXISTING.id)
+ // Nothing written: no second object, no second library row.
+ expect(send).not.toHaveBeenCalled()
+ expect(dbActions.createLibraryFile).not.toHaveBeenCalled()
+ })
+
+ it('narrows the candidate by name, type and size', async () => {
+ vi.mocked(dbActions.findChatFileCandidates).mockResolvedValue([])
+
+ await POST(uploadRequest(9999))
+
+ expect(dbActions.findChatFileCandidates).toHaveBeenCalledWith({
+ userId: 'user-1',
+ chatKeyPrefix: 'user-1/chats/chat-1/',
+ filename: 'report.pdf',
+ mediaType: 'application/pdf',
+ size: 9999
+ })
+ expect(dbActions.createLibraryFile).toHaveBeenCalled()
+ })
+
+ it('uploads when nothing matches', async () => {
+ vi.mocked(dbActions.findChatFileCandidates).mockResolvedValue([])
+
+ const { file } = await (await POST(uploadRequest())).json()
+
+ expect(file.key).not.toBe(EXISTING.objectKey)
+ expect(send).toHaveBeenCalledOnce()
+ })
+
+ it('uploads when the candidate holds different bytes', async () => {
+ // Same name, same type, same size, different content: the digest is the
+ // only thing standing between the user and a silently substituted file.
+ vi.mocked(dbActions.findChatFileCandidates).mockResolvedValue([
+ EXISTING
+ ] as any)
+ vi.mocked(getObjectContentMd5).mockResolvedValue(
+ createHash('md5').update('something else').digest('hex')
+ )
+
+ const { file } = await (await POST(uploadRequest())).json()
+
+ expect(file.key).not.toBe(EXISTING.objectKey)
+ expect(send).toHaveBeenCalledOnce()
+ })
+
+ it('uploads when the stored object is gone or has no usable digest', async () => {
+ vi.mocked(dbActions.findChatFileCandidates).mockResolvedValue([
+ EXISTING
+ ] as any)
+ vi.mocked(getObjectContentMd5).mockResolvedValue(null)
+
+ const { file } = await (await POST(uploadRequest())).json()
+
+ expect(file.key).not.toBe(EXISTING.objectKey)
+ expect(send).toHaveBeenCalledOnce()
+ })
+
+ it('checks every candidate, not just the newest', async () => {
+ // Two same-sized versions of one file: matching only the newest would mint
+ // a new object every time the user alternates between them.
+ const older = { ...EXISTING, id: 'file-0', objectKey: 'older-key' }
+ const newer = { ...EXISTING, id: 'file-9', objectKey: 'newer-key' }
+ vi.mocked(dbActions.findChatFileCandidates).mockResolvedValue([
+ newer,
+ older
+ ] as any)
+ vi.mocked(getObjectContentMd5).mockImplementation(async key =>
+ key === older.objectKey
+ ? md5OfUpload()
+ : createHash('md5').update('another version').digest('hex')
+ )
+
+ const { file } = await (await POST(uploadRequest())).json()
+
+ expect(file.key).toBe(older.objectKey)
+ expect(send).not.toHaveBeenCalled()
+ })
+
+ it('uploads when the lookup fails', async () => {
+ vi.mocked(dbActions.findChatFileCandidates).mockRejectedValue(
+ new Error('db down')
+ )
+
+ const response = await POST(uploadRequest())
+
+ expect(response.status).toBe(200)
+ expect(send).toHaveBeenCalledOnce()
+ })
+
+ it('skips the lookup in anonymous mode, which has no library', async () => {
+ process.env.ENABLE_AUTH = 'false'
+
+ await POST(uploadRequest())
+
+ expect(dbActions.findChatFileCandidates).not.toHaveBeenCalled()
+ expect(send).toHaveBeenCalledOnce()
+ })
+})
diff --git a/app/api/upload/route.ts b/app/api/upload/route.ts
index f6afcb84..48884ee2 100644
--- a/app/api/upload/route.ts
+++ b/app/api/upload/route.ts
@@ -1,11 +1,14 @@
import { NextRequest, NextResponse } from 'next/server'
import { PutObjectCommand } from '@aws-sdk/client-s3'
+import { createHash } from 'node:crypto'
import { capture } from '@/lib/analytics/dispatch'
import { getCurrentUserId } from '@/lib/auth/get-current-user'
import * as dbActions from '@/lib/db/actions'
import {
+ getChatFileObjectKeyPrefix,
+ getObjectContentMd5,
getR2Client,
getSignedFileUrl,
isObjectStorageConfigured,
@@ -61,8 +64,21 @@ export async function POST(req: NextRequest) {
{ status: 400 }
)
}
- const result = await uploadFileToR2(file, userId, chatId)
- if (process.env.ENABLE_AUTH === 'false') {
+ const isAnonymous = process.env.ENABLE_AUTH === 'false'
+ const buffer = Buffer.from(await file.arrayBuffer())
+
+ if (!isAnonymous && chatId) {
+ const reused = await reuseExistingChatFile(file, buffer, userId, chatId)
+ if (reused) {
+ return NextResponse.json(
+ { success: true, file: reused },
+ { status: 200 }
+ )
+ }
+ }
+
+ const result = await uploadFileToR2(file, buffer, userId, chatId)
+ if (isAnonymous) {
return NextResponse.json({ success: true, file: result }, { status: 200 })
}
@@ -112,16 +128,81 @@ export async function POST(req: NextRequest) {
}
}
+/**
+ * Points a repeated upload at the copy already stored for this chat.
+ *
+ * Re-uploading the same file is the common way a conversation ends up carrying
+ * two of it, usually because the first copy went unmentioned in the reply. A
+ * fresh upload would mint a new object key, which reads as a different file
+ * everywhere downstream, so the model would be handed both copies on every
+ * later turn.
+ *
+ * Name, media type and size only narrow the search to a few stored candidates.
+ * The decision itself is a digest comparison: handing back a different file
+ * under the same name would be a silent substitution, which is a far worse
+ * outcome than the duplicate this avoids. Every candidate is compared, or two
+ * same-sized versions of one file would push each other out of the search and
+ * mint a new object on every upload.
+ *
+ * Returns null whenever no match can be confirmed, and the caller uploads.
+ */
+async function reuseExistingChatFile(
+ file: File,
+ buffer: Buffer,
+ userId: string,
+ chatId: string
+) {
+ try {
+ const candidates = await dbActions.findChatFileCandidates({
+ userId,
+ chatKeyPrefix: getChatFileObjectKeyPrefix(userId, chatId),
+ filename: file.name,
+ mediaType: file.type,
+ size: file.size
+ })
+ if (candidates.length === 0) return null
+
+ const uploadedMd5 = createHash('md5').update(buffer).digest('hex')
+
+ for (const candidate of candidates) {
+ const storedMd5 = await getObjectContentMd5(candidate.objectKey)
+ if (!storedMd5 || storedMd5 !== uploadedMd5) continue
+
+ const url = await getSignedFileUrl(candidate.objectKey)
+
+ return {
+ type: 'file',
+ filename: candidate.filename,
+ key: candidate.objectKey,
+ url,
+ mediaType: candidate.mediaType,
+ id: candidate.id,
+ size: file.size,
+ libraryFile: { ...candidate, key: candidate.objectKey, url }
+ }
+ }
+
+ return null
+ } catch (error) {
+ console.error('Duplicate upload lookup failed:', error)
+ return null
+ }
+}
+
function sanitizeFilename(filename: string) {
return filename.replace(/[^a-z0-9.\-_]/gi, '_').toLowerCase()
}
-async function uploadFileToR2(file: File, userId: string, chatId: string) {
+async function uploadFileToR2(
+ file: File,
+ buffer: Buffer,
+ userId: string,
+ chatId: string
+) {
const sanitizedFileName = sanitizeFilename(file.name)
- const filePath = `${userId}/chats/${chatId}/${Date.now()}-${sanitizedFileName}`
+ const filePath = `${getChatFileObjectKeyPrefix(userId, chatId)}${Date.now()}-${sanitizedFileName}`
try {
- const buffer = Buffer.from(await file.arrayBuffer())
const r2Client = getR2Client()
await r2Client.send(
diff --git a/lib/db/actions.ts b/lib/db/actions.ts
index edfcf9a9..3e6c7a8f 100644
--- a/lib/db/actions.ts
+++ b/lib/db/actions.ts
@@ -1,6 +1,17 @@
'use server'
-import { and, asc, desc, eq, gt, ilike, inArray, lt, or } from 'drizzle-orm'
+import {
+ and,
+ asc,
+ desc,
+ eq,
+ gt,
+ ilike,
+ inArray,
+ lt,
+ or,
+ sql
+} from 'drizzle-orm'
import type { UIMessage } from '@/lib/types/ai'
import type { PersistableUIMessage } from '@/lib/types/message-persistence'
@@ -518,6 +529,61 @@ export async function deleteUserNotes(
}
}
+const CHAT_FILE_CANDIDATE_LIMIT = 5
+
+function escapeLikePattern(value: string) {
+ return value.replace(/[\\%_]/g, char => `\\${char}`)
+}
+
+/**
+ * Files already stored under this chat's object prefix that could be the one
+ * being uploaded, newest first.
+ *
+ * Candidates only: name, media type and size are metadata, so the caller has to
+ * confirm the bytes. More than one can match, and returning just the newest
+ * would keep re-uploading two same-sized versions of a file against each other.
+ *
+ * Matched on the object key prefix rather than on `chat_id`, because the column
+ * is not reliable for this. A file attached to a brand-new chat is uploaded
+ * before the chat row exists, and `createLibraryFile` stores it with a null
+ * `chat_id`; deleting a chat nulls it too. The key always carries the chat it
+ * was uploaded under.
+ *
+ * Staying inside one chat's prefix is the point: a chat deletion wipes that
+ * prefix, so reusing a key across chats would let one chat's deletion break
+ * another's attachment.
+ */
+export async function findChatFileCandidates({
+ userId,
+ chatKeyPrefix,
+ filename,
+ mediaType,
+ size
+}: {
+ userId: string
+ chatKeyPrefix: string
+ filename: string
+ mediaType: string
+ size: number
+}): Promise<LibraryFile[]> {
+ return withRLS(userId, async tx =>
+ tx
+ .select()
+ .from(libraryFiles)
+ .where(
+ and(
+ eq(libraryFiles.userId, userId),
+ eq(libraryFiles.filename, filename),
+ eq(libraryFiles.mediaType, mediaType),
+ eq(libraryFiles.size, size),
+ sql`${libraryFiles.objectKey} LIKE ${`${escapeLikePattern(chatKeyPrefix)}%`} ESCAPE '\\'`
+ )
+ )
+ .orderBy(desc(libraryFiles.createdAt))
+ .limit(CHAT_FILE_CANDIDATE_LIMIT)
+ )
+}
+
export async function createLibraryFile(
file: Omit<NewLibraryFile, 'id' | 'createdAt' | 'updatedAt'>
): Promise<LibraryFile> {
diff --git a/lib/storage/__tests__/r2-client.test.ts b/lib/storage/__tests__/r2-client.test.ts
index 2898ff59..9f7e7e0b 100644
--- a/lib/storage/__tests__/r2-client.test.ts
+++ b/lib/storage/__tests__/r2-client.test.ts
@@ -30,9 +30,18 @@ const s3Mocks = vi.hoisted(() => {
}
}
+ class HeadObjectCommand {
+ input: unknown
+
+ constructor(input: unknown) {
+ this.input = input
+ }
+ }
+
return {
DeleteObjectsCommand,
GetObjectCommand,
+ HeadObjectCommand,
ListObjectsV2Command,
S3Client,
getSignedUrl: vi.fn(),
@@ -43,6 +52,7 @@ const s3Mocks = vi.hoisted(() => {
vi.mock('@aws-sdk/client-s3', () => ({
DeleteObjectsCommand: s3Mocks.DeleteObjectsCommand,
GetObjectCommand: s3Mocks.GetObjectCommand,
+ HeadObjectCommand: s3Mocks.HeadObjectCommand,
ListObjectsV2Command: s3Mocks.ListObjectsV2Command,
S3Client: s3Mocks.S3Client
}))
@@ -168,6 +178,43 @@ describe('R2 client', () => {
})
})
+ describe('getObjectContentMd5', () => {
+ it('reads the digest off a single-part ETag', async () => {
+ const md5 = 'd41d8cd98f00b204e9800998ecf8427e'
+ s3Mocks.send.mockResolvedValueOnce({ ETag: `"${md5}"` })
+
+ const { getObjectContentMd5 } = await importR2Client()
+
+ await expect(getObjectContentMd5('user-id/file.pdf')).resolves.toBe(md5)
+ })
+
+ it('refuses an ETag that is not a content digest', async () => {
+ // A multipart or server-side-encrypted ETag is not the body's MD5, and
+ // treating it as one would compare two unrelated strings.
+ for (const etag of [
+ '"d41d8cd98f00b204e9800998ecf8427e-3"',
+ '"not-a-digest"',
+ undefined
+ ]) {
+ s3Mocks.send.mockResolvedValueOnce({ ETag: etag })
+
+ const { getObjectContentMd5 } = await importR2Client()
+
+ await expect(
+ getObjectContentMd5('user-id/file.pdf')
+ ).resolves.toBeNull()
+ }
+ })
+
+ it('answers null for a missing object', async () => {
+ s3Mocks.send.mockRejectedValueOnce(new Error('NotFound'))
+
+ const { getObjectContentMd5 } = await importR2Client()
+
+ await expect(getObjectContentMd5('user-id/gone.pdf')).resolves.toBeNull()
+ })
+ })
+
describe('signing date stability', () => {
beforeEach(() => {
vi.useFakeTimers()
diff --git a/lib/storage/r2-client.ts b/lib/storage/r2-client.ts
index 0f37fcb9..d35a2ea5 100644
--- a/lib/storage/r2-client.ts
+++ b/lib/storage/r2-client.ts
@@ -1,6 +1,7 @@
import {
DeleteObjectsCommand,
GetObjectCommand,
+ HeadObjectCommand,
ListObjectsV2Command,
S3Client
} from '@aws-sdk/client-s3'
@@ -164,6 +165,34 @@ export async function getSignedFileUrl(
)
}
+/**
+ * MD5 of a stored object's bytes, or null when it cannot be established.
+ *
+ * S3 and R2 set the ETag of a single-part upload to the MD5 of the body, which
+ * is the only content digest available here without re-downloading the object.
+ * Everything this app stores is written by one `PutObject` under a 5MB cap, so
+ * the ETag is that digest; a multipart ETag carries a `-<parts>` suffix and a
+ * server-side-encrypted one is opaque, and both answer null.
+ *
+ * Null also covers a missing object and a failed request: stored metadata can
+ * outlive the object it points at, and callers treat null as "cannot reuse".
+ */
+export async function getObjectContentMd5(key: string): Promise<string | null> {
+ const normalizedKey = normalizeObjectKey(key)
+ if (!normalizedKey) return null
+
+ try {
+ const head = await getR2Client().send(
+ new HeadObjectCommand({ Bucket: R2_BUCKET_NAME, Key: normalizedKey })
+ )
+ const etag = head.ETag?.replace(/"/g, '').toLowerCase() ?? ''
+
+ return /^[0-9a-f]{32}$/.test(etag) ? etag : null
+ } catch {
+ return null
+ }
+}
+
export async function signFilePartUrls(
parts: any[] = [],
options: SignFilePartUrlsOptions = {}
diff --git a/lib/streaming/create-chat-stream-response.ts b/lib/streaming/create-chat-stream-response.ts
index 6313e172..36bdaae1 100644
--- a/lib/streaming/create-chat-stream-response.ts
+++ b/lib/streaming/create-chat-stream-response.ts
@@ -25,6 +25,7 @@ import { capHistoricalAttachments } from './helpers/cap-historical-attachments'
import { compactHistoricalMessages } from './helpers/compact-historical-messages'
import { convertDataPart } from './helpers/convert-data-part'
import { assignDataPartNonces } from './helpers/data-part-nonce'
+import { dedupeAttachments } from './helpers/dedupe-attachments'
import { describeStreamError } from './helpers/describe-stream-error'
import {
EMPTY_RESPONSE_STATUS_MESSAGE,
@@ -149,8 +150,8 @@ export async function createChatStreamResponse(
const messagesWithNonces = assignDataPartNonces(messagesToModel)
const messagesWithoutSpec = stripSpecFromMessages(messagesWithNonces)
- const messagesToConvert = capHistoricalAttachments(
- compactHistoricalMessages(messagesWithoutSpec)
+ const messagesToConvert = dedupeAttachments(
+ capHistoricalAttachments(compactHistoricalMessages(messagesWithoutSpec))
)
// Convert to model messages and apply context window management
diff --git a/lib/streaming/create-ephemeral-chat-stream-response.ts b/lib/streaming/create-ephemeral-chat-stream-response.ts
index efac3c1b..2b601d1c 100644
--- a/lib/streaming/create-ephemeral-chat-stream-response.ts
+++ b/lib/streaming/create-ephemeral-chat-stream-response.ts
@@ -21,6 +21,7 @@ import { capHistoricalAttachments } from './helpers/cap-historical-attachments'
import { compactHistoricalMessages } from './helpers/compact-historical-messages'
import { convertDataPart } from './helpers/convert-data-part'
import { assignDataPartNonces } from './helpers/data-part-nonce'
+import { dedupeAttachments } from './helpers/dedupe-attachments'
import { describeStreamError } from './helpers/describe-stream-error'
import {
EMPTY_RESPONSE_STATUS_MESSAGE,
@@ -86,8 +87,8 @@ export async function createEphemeralChatStreamResponse(
try {
const messagesWithNonces = assignDataPartNonces(messages)
const messagesWithoutSpec = stripSpecFromMessages(messagesWithNonces)
- const messagesToConvert = capHistoricalAttachments(
- compactHistoricalMessages(messagesWithoutSpec)
+ const messagesToConvert = dedupeAttachments(
+ capHistoricalAttachments(compactHistoricalMessages(messagesWithoutSpec))
)
let modelMessages = await convertToModelMessages(messagesToConvert, {
diff --git a/lib/streaming/helpers/__tests__/dedupe-attachments.test.ts b/lib/streaming/helpers/__tests__/dedupe-attachments.test.ts
new file mode 100644
index 00000000..1adfcad2
--- /dev/null
+++ b/lib/streaming/helpers/__tests__/dedupe-attachments.test.ts
@@ -0,0 +1,205 @@
+import type { UIMessage } from 'ai'
+import { describe, expect, it } from 'vitest'
+
+import { dedupeAttachments } from '../dedupe-attachments'
+
+type FileSpec = {
+ filename: string
+ key?: string
+ url?: string
+ mediaType?: string
+}
+
+function userTurn(id: string, files: FileSpec[], text = 'Look at this') {
+ return {
+ id,
+ role: 'user',
+ parts: [
+ ...files.map(file => ({
+ type: 'file' as const,
+ mediaType: file.mediaType ?? 'application/pdf',
+ filename: file.filename,
+ url: file.url ?? `https://uploads.example.com/${file.key}?sig=abc`,
+ key: file.key
+ })),
+ { type: 'text', text }
+ ]
+ } as unknown as UIMessage
+}
+
+function assistantTurn(id: string): UIMessage {
+ return {
+ id,
+ role: 'assistant',
+ parts: [{ type: 'text', text: 'Here is what I see.' }]
+ } as unknown as UIMessage
+}
+
+function filenamesReaching(messages: UIMessage[]): string[] {
+ return messages.flatMap(message =>
+ message.parts
+ .filter(part => part.type === 'file')
+ .map(part => (part as unknown as { filename: string }).filename)
+ )
+}
+
+function placeholders(messages: UIMessage[]): string[] {
+ return messages.flatMap(message =>
+ message.parts
+ .filter(
+ part =>
+ part.type === 'text' &&
+ (part as unknown as { text: string }).text.startsWith(
+ '[Duplicate attachment omitted'
+ )
+ )
+ .map(part => (part as unknown as { text: string }).text)
+ )
+}
+
+describe('dedupeAttachments', () => {
+ it('leaves a conversation without duplicates untouched', () => {
+ const messages = [
+ userTurn('u0', [{ filename: 'a.pdf', key: 'user/chats/c/1-a.pdf' }]),
+ assistantTurn('a0'),
+ userTurn('u1', [{ filename: 'b.pdf', key: 'user/chats/c/2-b.pdf' }])
+ ]
+
+ expect(dedupeAttachments(messages)).toEqual(messages)
+ })
+
+ it('sends a re-attached file once, keeping the first occurrence', () => {
+ const key = 'user/chats/c/1-report.pdf'
+ const messages = [
+ userTurn('u0', [{ filename: 'report.pdf', key }]),
+ assistantTurn('a0'),
+ userTurn('u1', [{ filename: 'report.pdf', key }], 'Did you see it?')
+ ]
+
+ const deduped = dedupeAttachments(messages)
+
+ expect(filenamesReaching(deduped)).toEqual(['report.pdf'])
+ expect(deduped[0]).toEqual(messages[0])
+ expect(placeholders(deduped)).toHaveLength(1)
+ })
+
+ it('collapses copies attached twice within one message', () => {
+ const key = 'user/chats/c/1-report.pdf'
+ const messages = [
+ userTurn('u0', [
+ { filename: 'report.pdf', key },
+ { filename: 'report.pdf', key }
+ ])
+ ]
+
+ expect(filenamesReaching(dedupeAttachments(messages))).toHaveLength(1)
+ })
+
+ it('does not collapse distinct files that share a name', () => {
+ const messages = [
+ userTurn('u0', [
+ { filename: 'screenshot.png', key: 'user/chats/c/1-screenshot.png' }
+ ]),
+ assistantTurn('a0'),
+ userTurn('u1', [
+ { filename: 'screenshot.png', key: 'user/chats/c/2-screenshot.png' }
+ ])
+ ]
+
+ expect(dedupeAttachments(messages)).toEqual(messages)
+ })
+
+ it('falls back to the url when the part carries no object key', () => {
+ const url = 'data:image/png;base64,AAAA'
+ const messages = [
+ userTurn('u0', [{ filename: 'chart.png', url }]),
+ assistantTurn('a0'),
+ userTurn('u1', [{ filename: 'chart-copy.png', url }])
+ ]
+
+ expect(filenamesReaching(dedupeAttachments(messages))).toEqual([
+ 'chart.png'
+ ])
+ })
+
+ it('keeps parts that identify nothing', () => {
+ const messages = [
+ userTurn('u0', [{ filename: 'a.png', url: '' }]),
+ assistantTurn('a0'),
+ userTurn('u1', [{ filename: 'b.png', url: '' }])
+ ]
+
+ expect(dedupeAttachments(messages)).toEqual(messages)
+ })
+
+ it('keeps a message that held only duplicated attachments non-empty', () => {
+ const key = 'user/chats/c/1-only.png'
+ const messages: UIMessage[] = [
+ userTurn('u0', [{ filename: 'only.png', key }]),
+ assistantTurn('a0'),
+ {
+ id: 'files-only',
+ role: 'user',
+ parts: [
+ {
+ type: 'file',
+ mediaType: 'image/png',
+ filename: 'only.png',
+ url: 'https://uploads.example.com/1-only.png?sig=abc',
+ key
+ }
+ ]
+ } as unknown as UIMessage
+ ]
+
+ const deduped = dedupeAttachments(messages)
+
+ expect(deduped[2].parts).toHaveLength(1)
+ expect(deduped[2].parts[0]).toMatchObject({ type: 'text' })
+ })
+
+ it('keeps what was already sent identical as the thread grows', () => {
+ // The cache property: a later turn must not change the decision made for an
+ // earlier one, or the prompt prefix is rewritten on every turn.
+ const key = 'user/chats/c/1-report.pdf'
+ const head = [
+ userTurn('u0', [{ filename: 'report.pdf', key }]),
+ assistantTurn('a0'),
+ userTurn('u1', [{ filename: 'report.pdf', key }])
+ ]
+
+ const before = dedupeAttachments(head)
+ const after = dedupeAttachments([
+ ...head,
+ assistantTurn('a1'),
+ userTurn('u2', [{ filename: 'report.pdf', key }])
+ ])
+
+ expect(after.slice(0, head.length)).toEqual(before)
+ })
+
+ it('puts no url or timestamp in the placeholder and escapes the filename', () => {
+ const key = 'user/chats/c/1-hostile.png'
+ const hostile = '<script>alert(1)</script>\nsecond line.png'
+ const messages = [
+ userTurn('u0', [{ filename: hostile, key, mediaType: 'image/png' }]),
+ assistantTurn('a0'),
+ userTurn('u1', [
+ {
+ filename: hostile,
+ key,
+ mediaType: 'image/png',
+ url: 'https://uploads.example.com/x.png?X-Amz-Signature=deadbeef'
+ }
+ ])
+ ]
+
+ const [placeholder] = placeholders(dedupeAttachments(messages))
+
+ expect(placeholder).toContain('&lt;script&gt;')
+ expect(placeholder).not.toContain('<script>')
+ expect(placeholder).not.toContain('\n')
+ expect(placeholder).not.toContain('X-Amz-Signature')
+ expect(placeholder).not.toContain('https://')
+ })
+})
diff --git a/lib/streaming/helpers/attachment-parts.ts b/lib/streaming/helpers/attachment-parts.ts
new file mode 100644
index 00000000..d97b83b5
--- /dev/null
+++ b/lib/streaming/helpers/attachment-parts.ts
@@ -0,0 +1,30 @@
+import type { UIMessage } from 'ai'
+
+const MAX_FILENAME_CHARS = 80
+
+export type FilePartLike = {
+ mediaType?: string
+ filename?: string
+ url?: string
+ key?: string
+}
+
+export function isFilePart(part: UIMessage['parts'][number]) {
+ return part.type === 'file'
+}
+
+export function describeAttachment(part: FilePartLike) {
+ // The filename is user-supplied, so neutralize it the way source excerpts are
+ // neutralized: no markup, no newlines, bounded length.
+ const filename = (part.filename ?? '')
+ .replace(/</g, '&lt;')
+ .replace(/>/g, '&gt;')
+ .replace(/\s+/g, ' ')
+ .trim()
+ .slice(0, MAX_FILENAME_CHARS)
+ const mediaType = (part.mediaType ?? 'file').replace(/[^\w.+/-]/g, '')
+
+ return filename
+ ? `"${filename}" (${mediaType})`
+ : `an earlier ${mediaType} attachment`
+}
diff --git a/lib/streaming/helpers/cap-historical-attachments.ts b/lib/streaming/helpers/cap-historical-attachments.ts
index cf75404d..dd2ba1cd 100644
--- a/lib/streaming/helpers/cap-historical-attachments.ts
+++ b/lib/streaming/helpers/cap-historical-attachments.ts
@@ -1,5 +1,7 @@
import type { UIMessage } from 'ai'
+import { describeAttachment, isFilePart } from './attachment-parts'
+
const DEFAULT_REPLAY_LIMIT = 10
/**
@@ -24,24 +26,6 @@ export const HISTORY_ATTACHMENT_REPLAY_LIMIT = parseReplayLimit(
process.env.HISTORY_ATTACHMENT_REPLAY_LIMIT
)
-const MAX_FILENAME_CHARS = 80
-
-function describeAttachment(part: { mediaType?: string; filename?: string }) {
- // The filename is user-supplied, so neutralize it the way source excerpts are
- // neutralized: no markup, no newlines, bounded length.
- const filename = (part.filename ?? '')
- .replace(/</g, '&lt;')
- .replace(/>/g, '&gt;')
- .replace(/\s+/g, ' ')
- .trim()
- .slice(0, MAX_FILENAME_CHARS)
- const mediaType = (part.mediaType ?? 'file').replace(/[^\w.+/-]/g, '')
-
- return filename
- ? `"${filename}" (${mediaType})`
- : `an earlier ${mediaType} attachment`
-}
-
/**
* Number of leading attachments to drop, quantized to whole blocks of `limit`.
*
@@ -60,10 +44,6 @@ function getDroppedCount(total: number, limit: number): number {
return Math.floor((total - limit) / limit) * limit
}
-function isFilePart(part: UIMessage['parts'][number]) {
- return part.type === 'file'
-}
-
/**
* Caps how many of a conversation's attachments are replayed to the model.
*
diff --git a/lib/streaming/helpers/dedupe-attachments.ts b/lib/streaming/helpers/dedupe-attachments.ts
new file mode 100644
index 00000000..64353faf
--- /dev/null
+++ b/lib/streaming/helpers/dedupe-attachments.ts
@@ -0,0 +1,76 @@
+import type { UIMessage } from 'ai'
+
+import {
+ describeAttachment,
+ type FilePartLike,
+ isFilePart
+} from './attachment-parts'
+
+/**
+ * Identity of the stored bytes, or undefined when the part carries nothing that
+ * identifies them.
+ *
+ * The R2 object key names one immutable object, so two parts sharing it are the
+ * same file. The url is the fallback for parts stored without a key (data urls,
+ * external links); signed urls are derived from the key and signed once per
+ * request, so equal keys still produce equal urls within a single call.
+ *
+ * The filename is deliberately not part of the identity: two different files
+ * are routinely both called screenshot.png.
+ */
+function attachmentIdentity(part: FilePartLike): string | undefined {
+ if (part.key) return `key:${part.key}`
+ if (part.url) return `url:${part.url}`
+ return undefined
+}
+
+/**
+ * Sends a file to the model once, however many times it was attached.
+ *
+ * The same file lands in a conversation twice more often than it sounds: the
+ * usual path is a user re-attaching something because the first copy went
+ * unmentioned in the reply. Every copy is then replayed on every later turn,
+ * and a single PDF can be several hundred thousand tokens, so one accidental
+ * duplicate roughly doubles the input of the rest of the thread for no added
+ * information.
+ *
+ * The first occurrence is the one kept, so the decision for any given part
+ * depends only on the parts before it: appending turns never rewrites the
+ * prompt prefix, and the provider cache survives.
+ *
+ * This is not `capHistoricalAttachments`. That one bounds the attachment count
+ * and drops the oldest; duplicates can sit far below its cap and still dominate
+ * the payload. Run it first, so an attachment it already turned into a
+ * placeholder cannot swallow the copy the user just re-attached.
+ */
+export function dedupeAttachments(messages: UIMessage[]): UIMessage[] {
+ const seen = new Set<string>()
+
+ return messages.map(message => {
+ if (!message.parts.some(isFilePart)) return message
+
+ let replaced = false
+ const parts = message.parts.map(part => {
+ if (!isFilePart(part)) return part
+
+ const file = part as FilePartLike
+ const identity = attachmentIdentity(file)
+ if (!identity) return part
+
+ if (!seen.has(identity)) {
+ seen.add(identity)
+ return part
+ }
+
+ replaced = true
+ return {
+ type: 'text' as const,
+ text: `[Duplicate attachment omitted: ${describeAttachment(
+ file
+ )} is already attached earlier in this conversation.]`
+ }
+ })
+
+ return replaced ? { ...message, parts } : message
+ })
+}