An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
fix(attachments): deduplicate repeated attachments instead of replaying every copy (#947)
Yoshiki Miura346c30c453a1
diff --git a/app/api/upload/__tests__/route.test.ts b/app/api/upload/__tests__/route.test.tsnew file mode 100644index 00000000..776e4064--- /dev/null+++ b/app/api/upload/__tests__/route.test.ts@@ -0,0 +1,193 @@+import { beforeEach, describe, expect, it, vi } from 'vitest'++vi.mock('@/lib/auth/get-current-user', () => ({+ getCurrentUserId: vi.fn(() => Promise.resolve('user-1'))+}))++vi.mock('@/lib/analytics/dispatch', () => ({+ capture: vi.fn(() => Promise.resolve())+}))++vi.mock('@/lib/db/actions', () => ({+ findChatFileCandidates: vi.fn(),+ createLibraryFile: vi.fn()+}))++const send = vi.fn(() => Promise.resolve({}))++vi.mock('@/lib/storage/r2-client', () => ({+ getR2Client: vi.fn(() => ({ send })),+ getSignedFileUrl: vi.fn((key: string) =>+ Promise.resolve(`https://uploads.example.com/${key}?sig=abc`)+ ),+ isObjectStorageConfigured: vi.fn(() => true),+ getChatFileObjectKeyPrefix: (userId: string, chatId: string) =>+ `${userId}/chats/${chatId}/`,+ getObjectContentMd5: vi.fn(() => Promise.resolve<string | null>(null)),+ R2_BUCKET_NAME: 'test-bucket'+}))++import { createHash } from 'node:crypto'++import * as dbActions from '@/lib/db/actions'+import { getObjectContentMd5 } from '@/lib/storage/r2-client'++import { POST } from '../route'++const EXISTING = {+ id: 'file-1',+ userId: 'user-1',+ chatId: 'chat-1',+ filename: 'report.pdf',+ objectKey: 'user-1/chats/chat-1/1700000000000-report.pdf',+ mediaType: 'application/pdf',+ size: 1234,+ createdAt: new Date('2026-01-01'),+ updatedAt: new Date('2026-01-01')+}++/**+ * The route only reads headers and formData, so a stub avoids round-tripping a+ * File through multipart encoding in the test environment.+ */+function uploadRequest(bytes = 1234) {+ const file = {+ name: 'report.pdf',+ type: 'application/pdf',+ size: bytes,+ arrayBuffer: async () => new Uint8Array(bytes).buffer+ }+ const formData = new Map<string, unknown>([+ ['file', file],+ ['chatId', 'chat-1']+ ])++ return {+ headers: { get: () => 'multipart/form-data; boundary=x' },+ formData: async () => ({ get: (key: string) => formData.get(key) ?? null })+ } as any+}++function md5OfUpload(bytes = 1234) {+ return createHash('md5').update(Buffer.alloc(bytes)).digest('hex')+}++describe('POST /api/upload', () => {+ beforeEach(() => {+ vi.clearAllMocks()+ delete process.env.ENABLE_AUTH+ vi.mocked(getObjectContentMd5).mockResolvedValue(md5OfUpload())+ vi.mocked(dbActions.createLibraryFile).mockResolvedValue({+ ...EXISTING,+ id: 'file-2',+ objectKey: 'user-1/chats/chat-1/1800000000000-report.pdf'+ } as any)+ })++ it('reuses the stored copy when the same file is uploaded again', async () => {+ vi.mocked(dbActions.findChatFileCandidates).mockResolvedValue([+ EXISTING+ ] as any)++ const { file } = await (await POST(uploadRequest())).json()++ expect(file.key).toBe(EXISTING.objectKey)+ expect(file.id).toBe(EXISTING.id)+ // Nothing written: no second object, no second library row.+ expect(send).not.toHaveBeenCalled()+ expect(dbActions.createLibraryFile).not.toHaveBeenCalled()+ })++ it('narrows the candidate by name, type and size', async () => {+ vi.mocked(dbActions.findChatFileCandidates).mockResolvedValue([])++ await POST(uploadRequest(9999))++ expect(dbActions.findChatFileCandidates).toHaveBeenCalledWith({+ userId: 'user-1',+ chatKeyPrefix: 'user-1/chats/chat-1/',+ filename: 'report.pdf',+ mediaType: 'application/pdf',+ size: 9999+ })+ expect(dbActions.createLibraryFile).toHaveBeenCalled()+ })++ it('uploads when nothing matches', async () => {+ vi.mocked(dbActions.findChatFileCandidates).mockResolvedValue([])++ const { file } = await (await POST(uploadRequest())).json()++ expect(file.key).not.toBe(EXISTING.objectKey)+ expect(send).toHaveBeenCalledOnce()+ })++ it('uploads when the candidate holds different bytes', async () => {+ // Same name, same type, same size, different content: the digest is the+ // only thing standing between the user and a silently substituted file.+ vi.mocked(dbActions.findChatFileCandidates).mockResolvedValue([+ EXISTING+ ] as any)+ vi.mocked(getObjectContentMd5).mockResolvedValue(+ createHash('md5').update('something else').digest('hex')+ )++ const { file } = await (await POST(uploadRequest())).json()++ expect(file.key).not.toBe(EXISTING.objectKey)+ expect(send).toHaveBeenCalledOnce()+ })++ it('uploads when the stored object is gone or has no usable digest', async () => {+ vi.mocked(dbActions.findChatFileCandidates).mockResolvedValue([+ EXISTING+ ] as any)+ vi.mocked(getObjectContentMd5).mockResolvedValue(null)++ const { file } = await (await POST(uploadRequest())).json()++ expect(file.key).not.toBe(EXISTING.objectKey)+ expect(send).toHaveBeenCalledOnce()+ })++ it('checks every candidate, not just the newest', async () => {+ // Two same-sized versions of one file: matching only the newest would mint+ // a new object every time the user alternates between them.+ const older = { ...EXISTING, id: 'file-0', objectKey: 'older-key' }+ const newer = { ...EXISTING, id: 'file-9', objectKey: 'newer-key' }+ vi.mocked(dbActions.findChatFileCandidates).mockResolvedValue([+ newer,+ older+ ] as any)+ vi.mocked(getObjectContentMd5).mockImplementation(async key =>+ key === older.objectKey+ ? md5OfUpload()+ : createHash('md5').update('another version').digest('hex')+ )++ const { file } = await (await POST(uploadRequest())).json()++ expect(file.key).toBe(older.objectKey)+ expect(send).not.toHaveBeenCalled()+ })++ it('uploads when the lookup fails', async () => {+ vi.mocked(dbActions.findChatFileCandidates).mockRejectedValue(+ new Error('db down')+ )++ const response = await POST(uploadRequest())++ expect(response.status).toBe(200)+ expect(send).toHaveBeenCalledOnce()+ })++ it('skips the lookup in anonymous mode, which has no library', async () => {+ process.env.ENABLE_AUTH = 'false'++ await POST(uploadRequest())++ expect(dbActions.findChatFileCandidates).not.toHaveBeenCalled()+ expect(send).toHaveBeenCalledOnce()+ })+})diff --git a/app/api/upload/route.ts b/app/api/upload/route.tsindex f6afcb84..48884ee2 100644--- a/app/api/upload/route.ts+++ b/app/api/upload/route.ts@@ -1,11 +1,14 @@import { NextRequest, NextResponse } from 'next/server'import { PutObjectCommand } from '@aws-sdk/client-s3'+import { createHash } from 'node:crypto'import { capture } from '@/lib/analytics/dispatch'import { getCurrentUserId } from '@/lib/auth/get-current-user'import * as dbActions from '@/lib/db/actions'import {+ getChatFileObjectKeyPrefix,+ getObjectContentMd5,getR2Client,getSignedFileUrl,isObjectStorageConfigured,@@ -61,8 +64,21 @@ export async function POST(req: NextRequest) {{ status: 400 })}- const result = await uploadFileToR2(file, userId, chatId)- if (process.env.ENABLE_AUTH === 'false') {+ const isAnonymous = process.env.ENABLE_AUTH === 'false'+ const buffer = Buffer.from(await file.arrayBuffer())++ if (!isAnonymous && chatId) {+ const reused = await reuseExistingChatFile(file, buffer, userId, chatId)+ if (reused) {+ return NextResponse.json(+ { success: true, file: reused },+ { status: 200 }+ )+ }+ }++ const result = await uploadFileToR2(file, buffer, userId, chatId)+ if (isAnonymous) {return NextResponse.json({ success: true, file: result }, { status: 200 })}@@ -112,16 +128,81 @@ export async function POST(req: NextRequest) {}}+/**+ * Points a repeated upload at the copy already stored for this chat.+ *+ * Re-uploading the same file is the common way a conversation ends up carrying+ * two of it, usually because the first copy went unmentioned in the reply. A+ * fresh upload would mint a new object key, which reads as a different file+ * everywhere downstream, so the model would be handed both copies on every+ * later turn.+ *+ * Name, media type and size only narrow the search to a few stored candidates.+ * The decision itself is a digest comparison: handing back a different file+ * under the same name would be a silent substitution, which is a far worse+ * outcome than the duplicate this avoids. Every candidate is compared, or two+ * same-sized versions of one file would push each other out of the search and+ * mint a new object on every upload.+ *+ * Returns null whenever no match can be confirmed, and the caller uploads.+ */+async function reuseExistingChatFile(+ file: File,+ buffer: Buffer,+ userId: string,+ chatId: string+) {+ try {+ const candidates = await dbActions.findChatFileCandidates({+ userId,+ chatKeyPrefix: getChatFileObjectKeyPrefix(userId, chatId),+ filename: file.name,+ mediaType: file.type,+ size: file.size+ })+ if (candidates.length === 0) return null++ const uploadedMd5 = createHash('md5').update(buffer).digest('hex')++ for (const candidate of candidates) {+ const storedMd5 = await getObjectContentMd5(candidate.objectKey)+ if (!storedMd5 || storedMd5 !== uploadedMd5) continue++ const url = await getSignedFileUrl(candidate.objectKey)++ return {+ type: 'file',+ filename: candidate.filename,+ key: candidate.objectKey,+ url,+ mediaType: candidate.mediaType,+ id: candidate.id,+ size: file.size,+ libraryFile: { ...candidate, key: candidate.objectKey, url }+ }+ }++ return null+ } catch (error) {+ console.error('Duplicate upload lookup failed:', error)+ return null+ }+}+function sanitizeFilename(filename: string) {return filename.replace(/[^a-z0-9.\-_]/gi, '_').toLowerCase()}-async function uploadFileToR2(file: File, userId: string, chatId: string) {+async function uploadFileToR2(+ file: File,+ buffer: Buffer,+ userId: string,+ chatId: string+) {const sanitizedFileName = sanitizeFilename(file.name)- const filePath = `${userId}/chats/${chatId}/${Date.now()}-${sanitizedFileName}`+ const filePath = `${getChatFileObjectKeyPrefix(userId, chatId)}${Date.now()}-${sanitizedFileName}`try {- const buffer = Buffer.from(await file.arrayBuffer())const r2Client = getR2Client()await r2Client.send(diff --git a/lib/db/actions.ts b/lib/db/actions.tsindex edfcf9a9..3e6c7a8f 100644--- a/lib/db/actions.ts+++ b/lib/db/actions.ts@@ -1,6 +1,17 @@'use server'-import { and, asc, desc, eq, gt, ilike, inArray, lt, or } from 'drizzle-orm'+import {+ and,+ asc,+ desc,+ eq,+ gt,+ ilike,+ inArray,+ lt,+ or,+ sql+} from 'drizzle-orm'import type { UIMessage } from '@/lib/types/ai'import type { PersistableUIMessage } from '@/lib/types/message-persistence'@@ -518,6 +529,61 @@ export async function deleteUserNotes(}}+const CHAT_FILE_CANDIDATE_LIMIT = 5++function escapeLikePattern(value: string) {+ return value.replace(/[\\%_]/g, char => `\\${char}`)+}++/**+ * Files already stored under this chat's object prefix that could be the one+ * being uploaded, newest first.+ *+ * Candidates only: name, media type and size are metadata, so the caller has to+ * confirm the bytes. More than one can match, and returning just the newest+ * would keep re-uploading two same-sized versions of a file against each other.+ *+ * Matched on the object key prefix rather than on `chat_id`, because the column+ * is not reliable for this. A file attached to a brand-new chat is uploaded+ * before the chat row exists, and `createLibraryFile` stores it with a null+ * `chat_id`; deleting a chat nulls it too. The key always carries the chat it+ * was uploaded under.+ *+ * Staying inside one chat's prefix is the point: a chat deletion wipes that+ * prefix, so reusing a key across chats would let one chat's deletion break+ * another's attachment.+ */+export async function findChatFileCandidates({+ userId,+ chatKeyPrefix,+ filename,+ mediaType,+ size+}: {+ userId: string+ chatKeyPrefix: string+ filename: string+ mediaType: string+ size: number+}): Promise<LibraryFile[]> {+ return withRLS(userId, async tx =>+ tx+ .select()+ .from(libraryFiles)+ .where(+ and(+ eq(libraryFiles.userId, userId),+ eq(libraryFiles.filename, filename),+ eq(libraryFiles.mediaType, mediaType),+ eq(libraryFiles.size, size),+ sql`${libraryFiles.objectKey} LIKE ${`${escapeLikePattern(chatKeyPrefix)}%`} ESCAPE '\\'`+ )+ )+ .orderBy(desc(libraryFiles.createdAt))+ .limit(CHAT_FILE_CANDIDATE_LIMIT)+ )+}+export async function createLibraryFile(file: Omit<NewLibraryFile, 'id' | 'createdAt' | 'updatedAt'>): Promise<LibraryFile> {diff --git a/lib/storage/__tests__/r2-client.test.ts b/lib/storage/__tests__/r2-client.test.tsindex 2898ff59..9f7e7e0b 100644--- a/lib/storage/__tests__/r2-client.test.ts+++ b/lib/storage/__tests__/r2-client.test.ts@@ -30,9 +30,18 @@ const s3Mocks = vi.hoisted(() => {}}+ class HeadObjectCommand {+ input: unknown++ constructor(input: unknown) {+ this.input = input+ }+ }+return {DeleteObjectsCommand,GetObjectCommand,+ HeadObjectCommand,ListObjectsV2Command,S3Client,getSignedUrl: vi.fn(),@@ -43,6 +52,7 @@ const s3Mocks = vi.hoisted(() => {vi.mock('@aws-sdk/client-s3', () => ({DeleteObjectsCommand: s3Mocks.DeleteObjectsCommand,GetObjectCommand: s3Mocks.GetObjectCommand,+ HeadObjectCommand: s3Mocks.HeadObjectCommand,ListObjectsV2Command: s3Mocks.ListObjectsV2Command,S3Client: s3Mocks.S3Client}))@@ -168,6 +178,43 @@ describe('R2 client', () => {})})+ describe('getObjectContentMd5', () => {+ it('reads the digest off a single-part ETag', async () => {+ const md5 = 'd41d8cd98f00b204e9800998ecf8427e'+ s3Mocks.send.mockResolvedValueOnce({ ETag: `"${md5}"` })++ const { getObjectContentMd5 } = await importR2Client()++ await expect(getObjectContentMd5('user-id/file.pdf')).resolves.toBe(md5)+ })++ it('refuses an ETag that is not a content digest', async () => {+ // A multipart or server-side-encrypted ETag is not the body's MD5, and+ // treating it as one would compare two unrelated strings.+ for (const etag of [+ '"d41d8cd98f00b204e9800998ecf8427e-3"',+ '"not-a-digest"',+ undefined+ ]) {+ s3Mocks.send.mockResolvedValueOnce({ ETag: etag })++ const { getObjectContentMd5 } = await importR2Client()++ await expect(+ getObjectContentMd5('user-id/file.pdf')+ ).resolves.toBeNull()+ }+ })++ it('answers null for a missing object', async () => {+ s3Mocks.send.mockRejectedValueOnce(new Error('NotFound'))++ const { getObjectContentMd5 } = await importR2Client()++ await expect(getObjectContentMd5('user-id/gone.pdf')).resolves.toBeNull()+ })+ })+describe('signing date stability', () => {beforeEach(() => {vi.useFakeTimers()diff --git a/lib/storage/r2-client.ts b/lib/storage/r2-client.tsindex 0f37fcb9..d35a2ea5 100644--- a/lib/storage/r2-client.ts+++ b/lib/storage/r2-client.ts@@ -1,6 +1,7 @@import {DeleteObjectsCommand,GetObjectCommand,+ HeadObjectCommand,ListObjectsV2Command,S3Client} from '@aws-sdk/client-s3'@@ -164,6 +165,34 @@ export async function getSignedFileUrl()}+/**+ * MD5 of a stored object's bytes, or null when it cannot be established.+ *+ * S3 and R2 set the ETag of a single-part upload to the MD5 of the body, which+ * is the only content digest available here without re-downloading the object.+ * Everything this app stores is written by one `PutObject` under a 5MB cap, so+ * the ETag is that digest; a multipart ETag carries a `-<parts>` suffix and a+ * server-side-encrypted one is opaque, and both answer null.+ *+ * Null also covers a missing object and a failed request: stored metadata can+ * outlive the object it points at, and callers treat null as "cannot reuse".+ */+export async function getObjectContentMd5(key: string): Promise<string | null> {+ const normalizedKey = normalizeObjectKey(key)+ if (!normalizedKey) return null++ try {+ const head = await getR2Client().send(+ new HeadObjectCommand({ Bucket: R2_BUCKET_NAME, Key: normalizedKey })+ )+ const etag = head.ETag?.replace(/"/g, '').toLowerCase() ?? ''++ return /^[0-9a-f]{32}$/.test(etag) ? etag : null+ } catch {+ return null+ }+}+export async function signFilePartUrls(parts: any[] = [],options: SignFilePartUrlsOptions = {}diff --git a/lib/streaming/create-chat-stream-response.ts b/lib/streaming/create-chat-stream-response.tsindex 6313e172..36bdaae1 100644--- a/lib/streaming/create-chat-stream-response.ts+++ b/lib/streaming/create-chat-stream-response.ts@@ -25,6 +25,7 @@ import { capHistoricalAttachments } from './helpers/cap-historical-attachments'import { compactHistoricalMessages } from './helpers/compact-historical-messages'import { convertDataPart } from './helpers/convert-data-part'import { assignDataPartNonces } from './helpers/data-part-nonce'+import { dedupeAttachments } from './helpers/dedupe-attachments'import { describeStreamError } from './helpers/describe-stream-error'import {EMPTY_RESPONSE_STATUS_MESSAGE,@@ -149,8 +150,8 @@ export async function createChatStreamResponse(const messagesWithNonces = assignDataPartNonces(messagesToModel)const messagesWithoutSpec = stripSpecFromMessages(messagesWithNonces)- const messagesToConvert = capHistoricalAttachments(- compactHistoricalMessages(messagesWithoutSpec)+ const messagesToConvert = dedupeAttachments(+ capHistoricalAttachments(compactHistoricalMessages(messagesWithoutSpec)))// Convert to model messages and apply context window managementdiff --git a/lib/streaming/create-ephemeral-chat-stream-response.ts b/lib/streaming/create-ephemeral-chat-stream-response.tsindex efac3c1b..2b601d1c 100644--- a/lib/streaming/create-ephemeral-chat-stream-response.ts+++ b/lib/streaming/create-ephemeral-chat-stream-response.ts@@ -21,6 +21,7 @@ import { capHistoricalAttachments } from './helpers/cap-historical-attachments'import { compactHistoricalMessages } from './helpers/compact-historical-messages'import { convertDataPart } from './helpers/convert-data-part'import { assignDataPartNonces } from './helpers/data-part-nonce'+import { dedupeAttachments } from './helpers/dedupe-attachments'import { describeStreamError } from './helpers/describe-stream-error'import {EMPTY_RESPONSE_STATUS_MESSAGE,@@ -86,8 +87,8 @@ export async function createEphemeralChatStreamResponse(try {const messagesWithNonces = assignDataPartNonces(messages)const messagesWithoutSpec = stripSpecFromMessages(messagesWithNonces)- const messagesToConvert = capHistoricalAttachments(- compactHistoricalMessages(messagesWithoutSpec)+ const messagesToConvert = dedupeAttachments(+ capHistoricalAttachments(compactHistoricalMessages(messagesWithoutSpec)))let modelMessages = await convertToModelMessages(messagesToConvert, {diff --git a/lib/streaming/helpers/__tests__/dedupe-attachments.test.ts b/lib/streaming/helpers/__tests__/dedupe-attachments.test.tsnew file mode 100644index 00000000..1adfcad2--- /dev/null+++ b/lib/streaming/helpers/__tests__/dedupe-attachments.test.ts@@ -0,0 +1,205 @@+import type { UIMessage } from 'ai'+import { describe, expect, it } from 'vitest'++import { dedupeAttachments } from '../dedupe-attachments'++type FileSpec = {+ filename: string+ key?: string+ url?: string+ mediaType?: string+}++function userTurn(id: string, files: FileSpec[], text = 'Look at this') {+ return {+ id,+ role: 'user',+ parts: [+ ...files.map(file => ({+ type: 'file' as const,+ mediaType: file.mediaType ?? 'application/pdf',+ filename: file.filename,+ url: file.url ?? `https://uploads.example.com/${file.key}?sig=abc`,+ key: file.key+ })),+ { type: 'text', text }+ ]+ } as unknown as UIMessage+}++function assistantTurn(id: string): UIMessage {+ return {+ id,+ role: 'assistant',+ parts: [{ type: 'text', text: 'Here is what I see.' }]+ } as unknown as UIMessage+}++function filenamesReaching(messages: UIMessage[]): string[] {+ return messages.flatMap(message =>+ message.parts+ .filter(part => part.type === 'file')+ .map(part => (part as unknown as { filename: string }).filename)+ )+}++function placeholders(messages: UIMessage[]): string[] {+ return messages.flatMap(message =>+ message.parts+ .filter(+ part =>+ part.type === 'text' &&+ (part as unknown as { text: string }).text.startsWith(+ '[Duplicate attachment omitted'+ )+ )+ .map(part => (part as unknown as { text: string }).text)+ )+}++describe('dedupeAttachments', () => {+ it('leaves a conversation without duplicates untouched', () => {+ const messages = [+ userTurn('u0', [{ filename: 'a.pdf', key: 'user/chats/c/1-a.pdf' }]),+ assistantTurn('a0'),+ userTurn('u1', [{ filename: 'b.pdf', key: 'user/chats/c/2-b.pdf' }])+ ]++ expect(dedupeAttachments(messages)).toEqual(messages)+ })++ it('sends a re-attached file once, keeping the first occurrence', () => {+ const key = 'user/chats/c/1-report.pdf'+ const messages = [+ userTurn('u0', [{ filename: 'report.pdf', key }]),+ assistantTurn('a0'),+ userTurn('u1', [{ filename: 'report.pdf', key }], 'Did you see it?')+ ]++ const deduped = dedupeAttachments(messages)++ expect(filenamesReaching(deduped)).toEqual(['report.pdf'])+ expect(deduped[0]).toEqual(messages[0])+ expect(placeholders(deduped)).toHaveLength(1)+ })++ it('collapses copies attached twice within one message', () => {+ const key = 'user/chats/c/1-report.pdf'+ const messages = [+ userTurn('u0', [+ { filename: 'report.pdf', key },+ { filename: 'report.pdf', key }+ ])+ ]++ expect(filenamesReaching(dedupeAttachments(messages))).toHaveLength(1)+ })++ it('does not collapse distinct files that share a name', () => {+ const messages = [+ userTurn('u0', [+ { filename: 'screenshot.png', key: 'user/chats/c/1-screenshot.png' }+ ]),+ assistantTurn('a0'),+ userTurn('u1', [+ { filename: 'screenshot.png', key: 'user/chats/c/2-screenshot.png' }+ ])+ ]++ expect(dedupeAttachments(messages)).toEqual(messages)+ })++ it('falls back to the url when the part carries no object key', () => {+ const url = 'data:image/png;base64,AAAA'+ const messages = [+ userTurn('u0', [{ filename: 'chart.png', url }]),+ assistantTurn('a0'),+ userTurn('u1', [{ filename: 'chart-copy.png', url }])+ ]++ expect(filenamesReaching(dedupeAttachments(messages))).toEqual([+ 'chart.png'+ ])+ })++ it('keeps parts that identify nothing', () => {+ const messages = [+ userTurn('u0', [{ filename: 'a.png', url: '' }]),+ assistantTurn('a0'),+ userTurn('u1', [{ filename: 'b.png', url: '' }])+ ]++ expect(dedupeAttachments(messages)).toEqual(messages)+ })++ it('keeps a message that held only duplicated attachments non-empty', () => {+ const key = 'user/chats/c/1-only.png'+ const messages: UIMessage[] = [+ userTurn('u0', [{ filename: 'only.png', key }]),+ assistantTurn('a0'),+ {+ id: 'files-only',+ role: 'user',+ parts: [+ {+ type: 'file',+ mediaType: 'image/png',+ filename: 'only.png',+ url: 'https://uploads.example.com/1-only.png?sig=abc',+ key+ }+ ]+ } as unknown as UIMessage+ ]++ const deduped = dedupeAttachments(messages)++ expect(deduped[2].parts).toHaveLength(1)+ expect(deduped[2].parts[0]).toMatchObject({ type: 'text' })+ })++ it('keeps what was already sent identical as the thread grows', () => {+ // The cache property: a later turn must not change the decision made for an+ // earlier one, or the prompt prefix is rewritten on every turn.+ const key = 'user/chats/c/1-report.pdf'+ const head = [+ userTurn('u0', [{ filename: 'report.pdf', key }]),+ assistantTurn('a0'),+ userTurn('u1', [{ filename: 'report.pdf', key }])+ ]++ const before = dedupeAttachments(head)+ const after = dedupeAttachments([+ ...head,+ assistantTurn('a1'),+ userTurn('u2', [{ filename: 'report.pdf', key }])+ ])++ expect(after.slice(0, head.length)).toEqual(before)+ })++ it('puts no url or timestamp in the placeholder and escapes the filename', () => {+ const key = 'user/chats/c/1-hostile.png'+ const hostile = '<script>alert(1)</script>\nsecond line.png'+ const messages = [+ userTurn('u0', [{ filename: hostile, key, mediaType: 'image/png' }]),+ assistantTurn('a0'),+ userTurn('u1', [+ {+ filename: hostile,+ key,+ mediaType: 'image/png',+ url: 'https://uploads.example.com/x.png?X-Amz-Signature=deadbeef'+ }+ ])+ ]++ const [placeholder] = placeholders(dedupeAttachments(messages))++ expect(placeholder).toContain('<script>')+ expect(placeholder).not.toContain('<script>')+ expect(placeholder).not.toContain('\n')+ expect(placeholder).not.toContain('X-Amz-Signature')+ expect(placeholder).not.toContain('https://')+ })+})diff --git a/lib/streaming/helpers/attachment-parts.ts b/lib/streaming/helpers/attachment-parts.tsnew file mode 100644index 00000000..d97b83b5--- /dev/null+++ b/lib/streaming/helpers/attachment-parts.ts@@ -0,0 +1,30 @@+import type { UIMessage } from 'ai'++const MAX_FILENAME_CHARS = 80++export type FilePartLike = {+ mediaType?: string+ filename?: string+ url?: string+ key?: string+}++export function isFilePart(part: UIMessage['parts'][number]) {+ return part.type === 'file'+}++export function describeAttachment(part: FilePartLike) {+ // The filename is user-supplied, so neutralize it the way source excerpts are+ // neutralized: no markup, no newlines, bounded length.+ const filename = (part.filename ?? '')+ .replace(/</g, '<')+ .replace(/>/g, '>')+ .replace(/\s+/g, ' ')+ .trim()+ .slice(0, MAX_FILENAME_CHARS)+ const mediaType = (part.mediaType ?? 'file').replace(/[^\w.+/-]/g, '')++ return filename+ ? `"${filename}" (${mediaType})`+ : `an earlier ${mediaType} attachment`+}diff --git a/lib/streaming/helpers/cap-historical-attachments.ts b/lib/streaming/helpers/cap-historical-attachments.tsindex cf75404d..dd2ba1cd 100644--- a/lib/streaming/helpers/cap-historical-attachments.ts+++ b/lib/streaming/helpers/cap-historical-attachments.ts@@ -1,5 +1,7 @@import type { UIMessage } from 'ai'+import { describeAttachment, isFilePart } from './attachment-parts'+const DEFAULT_REPLAY_LIMIT = 10/**@@ -24,24 +26,6 @@ export const HISTORY_ATTACHMENT_REPLAY_LIMIT = parseReplayLimit(process.env.HISTORY_ATTACHMENT_REPLAY_LIMIT)-const MAX_FILENAME_CHARS = 80--function describeAttachment(part: { mediaType?: string; filename?: string }) {- // The filename is user-supplied, so neutralize it the way source excerpts are- // neutralized: no markup, no newlines, bounded length.- const filename = (part.filename ?? '')- .replace(/</g, '<')- .replace(/>/g, '>')- .replace(/\s+/g, ' ')- .trim()- .slice(0, MAX_FILENAME_CHARS)- const mediaType = (part.mediaType ?? 'file').replace(/[^\w.+/-]/g, '')-- return filename- ? `"${filename}" (${mediaType})`- : `an earlier ${mediaType} attachment`-}-/*** Number of leading attachments to drop, quantized to whole blocks of `limit`.*@@ -60,10 +44,6 @@ function getDroppedCount(total: number, limit: number): number {return Math.floor((total - limit) / limit) * limit}-function isFilePart(part: UIMessage['parts'][number]) {- return part.type === 'file'-}-/*** Caps how many of a conversation's attachments are replayed to the model.*diff --git a/lib/streaming/helpers/dedupe-attachments.ts b/lib/streaming/helpers/dedupe-attachments.tsnew file mode 100644index 00000000..64353faf--- /dev/null+++ b/lib/streaming/helpers/dedupe-attachments.ts@@ -0,0 +1,76 @@+import type { UIMessage } from 'ai'++import {+ describeAttachment,+ type FilePartLike,+ isFilePart+} from './attachment-parts'++/**+ * Identity of the stored bytes, or undefined when the part carries nothing that+ * identifies them.+ *+ * The R2 object key names one immutable object, so two parts sharing it are the+ * same file. The url is the fallback for parts stored without a key (data urls,+ * external links); signed urls are derived from the key and signed once per+ * request, so equal keys still produce equal urls within a single call.+ *+ * The filename is deliberately not part of the identity: two different files+ * are routinely both called screenshot.png.+ */+function attachmentIdentity(part: FilePartLike): string | undefined {+ if (part.key) return `key:${part.key}`+ if (part.url) return `url:${part.url}`+ return undefined+}++/**+ * Sends a file to the model once, however many times it was attached.+ *+ * The same file lands in a conversation twice more often than it sounds: the+ * usual path is a user re-attaching something because the first copy went+ * unmentioned in the reply. Every copy is then replayed on every later turn,+ * and a single PDF can be several hundred thousand tokens, so one accidental+ * duplicate roughly doubles the input of the rest of the thread for no added+ * information.+ *+ * The first occurrence is the one kept, so the decision for any given part+ * depends only on the parts before it: appending turns never rewrites the+ * prompt prefix, and the provider cache survives.+ *+ * This is not `capHistoricalAttachments`. That one bounds the attachment count+ * and drops the oldest; duplicates can sit far below its cap and still dominate+ * the payload. Run it first, so an attachment it already turned into a+ * placeholder cannot swallow the copy the user just re-attached.+ */+export function dedupeAttachments(messages: UIMessage[]): UIMessage[] {+ const seen = new Set<string>()++ return messages.map(message => {+ if (!message.parts.some(isFilePart)) return message++ let replaced = false+ const parts = message.parts.map(part => {+ if (!isFilePart(part)) return part++ const file = part as FilePartLike+ const identity = attachmentIdentity(file)+ if (!identity) return part++ if (!seen.has(identity)) {+ seen.add(identity)+ return part+ }++ replaced = true+ return {+ type: 'text' as const,+ text: `[Duplicate attachment omitted: ${describeAttachment(+ file+ )} is already attached earlier in this conversation.]`+ }+ })++ return replaced ? { ...message, parts } : message+ })+}