Papermark is the open-source DocSend alternative and secure data rooms with built-in analytics and custom domains.
feat(ee): improve agents
Marc Seitzf5c793e11577
diff --git a/app/(ee)/api/ai/chat/[chatId]/messages/route.ts b/app/(ee)/api/ai/chat/[chatId]/messages/route.tsindex c833d36b..f7e800c4 100644--- a/app/(ee)/api/ai/chat/[chatId]/messages/route.ts+++ b/app/(ee)/api/ai/chat/[chatId]/messages/route.ts@@ -35,8 +35,11 @@ export async function POST();}- const { content, filterDocumentId, filterDataroomDocumentIds } =- validation.data;+ const {+ content,+ filterDocumentId,+ filterDataroomDocumentIds,+ } = validation.data;const session = await getServerSession(authOptions);const searchParams = req.nextUrl.searchParams;@@ -133,16 +136,49 @@ export async function POST(}// Send message and get streaming response- const result = await sendMessage({+ const { result, referencesForStream } = await sendMessage({chatId,content,vectorStoreId: chat.vectorStoreId,filteredDataroomDocumentIds,filterDocumentId,userSelectedDataroomDocumentIds: filterDataroomDocumentIds,+ dataroomId: chat.dataroomId || undefined,+ linkId: chat.linkId || undefined,});- return result.toTextStreamResponse();+ const encoder = new TextEncoder();+ const stream = new ReadableStream<Uint8Array>({+ async start(controller) {+ try {+ for await (const chunk of result.textStream) {+ controller.enqueue(encoder.encode(chunk));+ }++ const referencesSection = await Promise.race([+ referencesForStream,+ new Promise<string>((resolve) =>+ setTimeout(() => resolve(""), 5000),+ ),+ ]);++ if (referencesSection) {+ controller.enqueue(encoder.encode(referencesSection));+ }++ controller.close();+ } catch (error) {+ console.error("Error streaming AI response:", error);+ controller.error(error);+ }+ },+ });++ return new Response(stream, {+ headers: {+ "Content-Type": "text/plain; charset=utf-8",+ },+ });} catch (error) {console.error("Error sending message:", error);return new Response(JSON.stringify({ error: "Internal server error" }), {diff --git a/ee/features/ai/components/chat-message.tsx b/ee/features/ai/components/chat-message.tsxindex b9adab04..6715f416 100644--- a/ee/features/ai/components/chat-message.tsx+++ b/ee/features/ai/components/chat-message.tsx@@ -1,14 +1,21 @@"use client";-import { memo, useState } from "react";+import { type ComponentPropsWithoutRef, memo, useMemo, useState } from "react";import {CheckIcon,+ ChevronDownIcon,CopyIcon,- ThumbsDownIcon,- ThumbsUpIcon,+ FileIcon,+ FileSpreadsheetIcon,+ FileTextIcon,+ FolderIcon,+ ImageIcon,+ PresentationIcon,} from "lucide-react";+import { cn } from "@/lib/utils";+import {Message,MessageAction,@@ -18,6 +25,8 @@ import {} from "@/components/ai-elements/message";import { Shimmer } from "@/components/ai-elements/shimmer";+import type { ChatStreamSource } from "../lib/chat/send-message";+// ============================================================================// Types// ============================================================================@@ -29,10 +38,214 @@ export interface ChatMessageProps {className?: string;messageId?: string;onFeedback?: (messageId: string, feedback: "up" | "down") => void;+ sources?: ChatStreamSource[];+ suggestedQuestions?: string[];+ onSuggestedQuestionClick?: (question: string) => void;+ isLastAssistantMessage?: boolean;}type FeedbackState = "up" | "down" | null;+function getViewerBasePath(): string {+ if (typeof window === "undefined") return "";+ const path = window.location.pathname;+ const dIndex = path.indexOf("/d/");+ return dIndex !== -1 ? path.slice(0, dIndex) : path;+}++function buildDocumentUrl(+ dataroomDocumentId: string,+ page?: number,+): string {+ const base = getViewerBasePath();+ const hash = page ? `#page=${page}` : "";+ return `${base}/d/${dataroomDocumentId}${hash}`;+}++function normalizeAssistantLinkHref(href?: string): string | undefined {+ if (!href) {+ return href;+ }++ if (href.startsWith("/")) {+ const dIndex = href.indexOf("/d/");+ if (dIndex !== -1) {+ const docSegment = href.slice(dIndex);+ return `${getViewerBasePath()}${docSegment}`;+ }+ return href;+ }++ try {+ const parsed = new URL(href);+ const dIndex = parsed.pathname.indexOf("/d/");+ if (dIndex !== -1) {+ const docSegment = parsed.pathname.slice(dIndex);+ return `${getViewerBasePath()}${docSegment}${parsed.search}${parsed.hash}`;+ }+ } catch {+ // Keep original href if URL parsing fails.+ }++ return href;+}++function AssistantLink({+ children,+ className,+ href,+ ...props+}: ComponentPropsWithoutRef<"a">) {+ const normalizedHref = normalizeAssistantLinkHref(href);++ return (+ <a+ {...props}+ className={cn("inline-flex items-center", className)}+ href={normalizedHref}+ rel="noopener noreferrer"+ target="_blank"+ >+ {children}+ </a>+ );+}++// ============================================================================+// File type icon helper+// ============================================================================++function getFileIcon(filename: string) {+ const ext = filename.split(".").pop()?.toLowerCase();+ switch (ext) {+ case "pdf":+ return <FileTextIcon className="size-3.5 text-red-500" />;+ case "ppt":+ case "pptx":+ return <PresentationIcon className="size-3.5 text-orange-500" />;+ case "xls":+ case "xlsx":+ case "csv":+ return <FileSpreadsheetIcon className="size-3.5 text-green-600" />;+ case "jpg":+ case "jpeg":+ case "png":+ case "gif":+ case "webp":+ return <ImageIcon className="size-3.5 text-purple-500" />;+ default:+ return <FileIcon className="size-3.5 text-blue-500" />;+ }+}++// ============================================================================+// Sources Section+// ============================================================================++function SourcesSection({ sources }: { sources: ChatStreamSource[] }) {+ const [expanded, setExpanded] = useState(false);++ if (sources.length === 0) return null;++ return (+ <div className="!mt-3 rounded-lg border border-gray-200 bg-gray-50/50">+ <button+ type="button"+ onClick={() => setExpanded(!expanded)}+ className="flex w-full items-center justify-between px-3 py-2 text-left text-xs font-medium text-gray-600 transition-colors hover:bg-gray-100/50"+ >+ <span>+ Source: {sources.length}{" "}+ {sources.length === 1 ? "document" : "documents"}+ </span>+ <ChevronDownIcon+ className={cn(+ "size-3.5 text-gray-400 transition-transform duration-200",+ expanded && "rotate-180",+ )}+ />+ </button>++ {expanded && (+ <div className="border-t border-gray-200 px-3 py-2">+ <div className="space-y-1.5">+ {sources.map((source) => (+ <a+ key={`${source.id}-${source.page}`}+ href={+ source.dataroomDocumentId+ ? buildDocumentUrl(source.dataroomDocumentId, source.page)+ : normalizeAssistantLinkHref(source.url)+ }+ target="_blank"+ rel="noopener noreferrer"+ className="group flex items-start gap-2 rounded-md px-2 py-1.5 transition-colors hover:bg-gray-100"+ >+ <span className="mt-0.5 shrink-0 text-[10px] font-semibold text-gray-400">+ {source.id}+ </span>+ <div className="min-w-0 flex-1">+ <div className="flex items-center gap-1.5">+ {getFileIcon(source.name)}+ <span className="truncate text-xs font-medium text-gray-900 group-hover:text-primary">+ {source.name}+ </span>+ {source.page && (+ <span className="shrink-0 rounded bg-gray-200 px-1.5 py-0.5 text-[10px] font-medium text-gray-600">+ Page {source.page}+ </span>+ )}+ </div>+ {source.folderPath && (+ <div className="mt-0.5 flex items-center gap-1 text-[10px] text-gray-400">+ <FolderIcon className="size-2.5" />+ <span className="truncate">{source.folderPath}</span>+ </div>+ )}+ </div>+ </a>+ ))}+ </div>+ </div>+ )}+ </div>+ );+}++// ============================================================================+// Suggested Questions+// ============================================================================++function SuggestedQuestions({+ questions,+ onQuestionClick,+}: {+ questions: string[];+ onQuestionClick?: (question: string) => void;+}) {+ if (questions.length === 0) return null;++ return (+ <div className="!mt-5">+ <p className="mb-1.5 text-[10px] font-medium uppercase tracking-wider text-gray-400">+ Suggested questions+ </p>+ <div className="space-y-1">+ {questions.map((question, idx) => (+ <button+ key={idx}+ type="button"+ onClick={() => onQuestionClick?.(question)}+ className="block w-full rounded-md border border-gray-200 bg-white px-3 py-2 text-left text-xs text-gray-700 transition-colors hover:border-primary/30 hover:bg-primary/5 hover:text-gray-900"+ >+ {question}+ </button>+ ))}+ </div>+ </div>+ );+}+// ============================================================================// Main Component// ============================================================================@@ -44,10 +257,20 @@ export const ChatMessage = memo(function ChatMessage({className,messageId,onFeedback,+ sources,+ suggestedQuestions,+ onSuggestedQuestionClick,+ isLastAssistantMessage,}: ChatMessageProps) {const [copied, setCopied] = useState(false);const [feedback, setFeedback] = useState<FeedbackState>(null);const isUser = role === "user";+ const responseComponents = useMemo(+ () => ({+ a: AssistantLink,+ }),+ [],+ );const handleCopy = async () => {try {@@ -60,7 +283,6 @@ export const ChatMessage = memo(function ChatMessage({};const handleFeedback = (type: "up" | "down") => {- // Toggle feedback if clicking the same button, otherwise set new feedbackconst newFeedback = feedback === type ? null : type;setFeedback(newFeedback);@@ -69,7 +291,6 @@ export const ChatMessage = memo(function ChatMessage({}};- // For user messages, simple renderingif (isUser) {return (<Message from="user" className={className}>@@ -80,7 +301,6 @@ export const ChatMessage = memo(function ChatMessage({);}- // For assistant messagesreturn (<div className="space-y-1"><Message from="assistant" className={className}>@@ -89,62 +309,49 @@ export const ChatMessage = memo(function ChatMessage({<Shimmer duration={1.5}>Generating response...</Shimmer>) : (<div className="prose prose-sm max-w-none dark:prose-invert">- <MessageResponse>{content}</MessageResponse>+ <MessageResponse+ components={responseComponents}+ linkSafety={{ enabled: false }}+ >+ {content}+ </MessageResponse></div>)}</MessageContent></Message>- {/* Message actions - always visible when there's content and not streaming */}{content && !isStreaming && (- <MessageActions className="ml-0">- <MessageAction- onClick={handleCopy}- tooltip={copied ? "Copied!" : "Copy"}- label={copied ? "Copied" : "Copy message"}- variant="ghost"- size="icon"- className="size-7"- >- {copied ? (- <CheckIcon className="size-3.5 text-green-500" />- ) : (- <CopyIcon className="size-3.5 text-muted-foreground" />+ <>+ <MessageActions className="ml-0">+ <MessageAction+ onClick={handleCopy}+ tooltip={copied ? "Copied!" : "Copy"}+ label={copied ? "Copied" : "Copy message"}+ variant="ghost"+ size="icon"+ className="size-7"+ >+ {copied ? (+ <CheckIcon className="size-3.5 text-green-500" />+ ) : (+ <CopyIcon className="size-3.5 text-muted-foreground" />+ )}+ </MessageAction>+ </MessageActions>++ {sources && sources.length > 0 && (+ <SourcesSection sources={sources} />+ )}++ {isLastAssistantMessage &&+ suggestedQuestions &&+ suggestedQuestions.length > 0 && (+ <SuggestedQuestions+ questions={suggestedQuestions}+ onQuestionClick={onSuggestedQuestionClick}+ />)}- </MessageAction>- {/* <MessageAction- onClick={() => handleFeedback("up")}- tooltip="Good response"- label="Good response"- variant="ghost"- size="icon"- className="size-7"- >- <ThumbsUpIcon- className={`size-3.5 ${- feedback === "up"- ? "fill-green-500 text-green-500"- : "text-muted-foreground"- }`}- />- </MessageAction>- <MessageAction- onClick={() => handleFeedback("down")}- tooltip="Bad response"- label="Bad response"- variant="ghost"- size="icon"- className="size-7"- >- <ThumbsDownIcon- className={`size-3.5 ${- feedback === "down"- ? "fill-red-500 text-red-500"- : "text-muted-foreground"- }`}- />- </MessageAction> */}- </MessageActions>+ </>)}</div>);diff --git a/ee/features/ai/components/viewer-chat-panel.tsx b/ee/features/ai/components/viewer-chat-panel.tsxindex aa5fe658..a6bccb0d 100644--- a/ee/features/ai/components/viewer-chat-panel.tsx+++ b/ee/features/ai/components/viewer-chat-panel.tsx@@ -6,8 +6,11 @@ import {AtSignIcon,FileTextIcon,Loader2,+ MessageSquareTextIcon,Plus,PlusIcon,+ SearchIcon,+ ShieldCheckIcon,SparklesIcon,XIcon,} from "lucide-react";@@ -36,6 +39,14 @@ import {TooltipTrigger,} from "@/components/ui/tooltip";+import type {+ ChatStreamMetadata,+ ChatStreamSource,+} from "../lib/chat/send-message";+import {+ CHAT_METADATA_PREFIX,+ CHAT_METADATA_SUFFIX,+} from "../lib/chat/send-message";import { parseTextStream } from "../lib/stream/parse-text-stream";import ChatMessage from "./chat-message";import {@@ -52,6 +63,67 @@ interface Message {role: "user" | "assistant";content: string;isStreaming?: boolean;+ sources?: ChatStreamSource[];+ suggestedQuestions?: string[];+}++function extractStreamMetadata(content: string): {+ text: string;+ metadata?: ChatStreamMetadata;+} {+ const prefixIdx = content.indexOf(CHAT_METADATA_PREFIX);+ if (prefixIdx === -1) return { text: content };++ const jsonStart = prefixIdx + CHAT_METADATA_PREFIX.length;+ const suffixIdx = content.indexOf(CHAT_METADATA_SUFFIX, jsonStart);+ if (suffixIdx === -1) return { text: content };++ try {+ const jsonStr = content.slice(jsonStart, suffixIdx);+ const metadata = JSON.parse(jsonStr) as ChatStreamMetadata;+ const text = content.slice(0, prefixIdx).trim();+ return { text, metadata };+ } catch {+ return { text: content };+ }+}++const REFERENCES_REGEX =+ /\n\n(?:#{1,6}\s*)?References\s*\n(?:\s*\n)?((?:[-*]\s.*(?:\n|$))+)$/i;+const REFERENCE_LINE_REGEX = /[-*]\s*\[(.+?)]\((.+?)\)(?:\s*-\s*p\.\s*(\d+))?/;++interface ResolvedRef {+ dataroomDocumentId: string;+ documentName: string;+ url: string;+ page?: number;+ folderPath?: string;+}++function parseMarkdownReferences(content: string): {+ text: string;+ sources?: ChatStreamSource[];+} {+ const match = content.match(REFERENCES_REGEX);+ if (!match) return { text: content };++ const text = content.replace(REFERENCES_REGEX, "").trim();+ const lines = match[1].trim().split("\n").filter(Boolean);+ const sources: ChatStreamSource[] = [];++ for (let i = 0; i < lines.length; i++) {+ const lineMatch = lines[i].match(REFERENCE_LINE_REGEX);+ if (lineMatch) {+ sources.push({+ id: `D-${i + 1}`,+ name: lineMatch[1].replace(/\\([\[\]])/g, "$1"),+ url: lineMatch[2],+ page: lineMatch[3] ? parseInt(lineMatch[3]) : undefined,+ });+ }+ }++ return { text, sources: sources.length > 0 ? sources : undefined };}interface ViewerChatPanelProps {@@ -246,12 +318,43 @@ function ViewerChatPanelContent({setChatId(selectedChatId);setChatTitle(data.title);- // Convert messages to the format expected by the UI- const loadedMessages: Message[] = data.messages.map((msg: any) => ({- id: msg.id,- role: msg.role,- content: msg.content,- }));+ const loadedMessages: Message[] = data.messages.map((msg: any) => {+ if (msg.role === "assistant") {+ const meta = msg.metadata as Record<string, any> | null;+ const storedSuggestions = meta?.suggestedQuestions as+ | string[]+ | undefined;+ const storedRefs = meta?.references as ResolvedRef[] | undefined;++ if (storedRefs && storedRefs.length > 0) {+ const { text } = parseMarkdownReferences(msg.content);+ return {+ id: msg.id,+ role: msg.role,+ content: text,+ sources: storedRefs.map((ref: ResolvedRef, idx: number) => ({+ id: `D-${idx + 1}`,+ name: ref.documentName,+ url: ref.url,+ dataroomDocumentId: ref.dataroomDocumentId,+ page: ref.page,+ folderPath: ref.folderPath,+ })),+ suggestedQuestions: storedSuggestions,+ };+ }++ const { text, sources } = parseMarkdownReferences(msg.content);+ return {+ id: msg.id,+ role: msg.role,+ content: text,+ sources,+ suggestedQuestions: storedSuggestions,+ };+ }+ return { id: msg.id, role: msg.role, content: msg.content };+ });setMessages(loadedMessages);} catch (error) {@@ -355,20 +458,27 @@ function ViewerChatPanelContent({};setMessages((prev) => [...prev, assistantMessage]);- // Parse the text streamawait parseTextStream(reader, {onTextDelta: (_delta, accumulated) => {+ const { text: visibleText } = extractStreamMetadata(accumulated);setMessages((prev) =>prev.map((m) =>- m.id === assistantMessageId ? { ...m, content: accumulated } : m,+ m.id === assistantMessageId ? { ...m, content: visibleText } : m,),);},- onTextEnd: (content) => {+ onTextEnd: (rawContent) => {+ const { text, metadata } = extractStreamMetadata(rawContent);setMessages((prev) =>prev.map((m) =>m.id === assistantMessageId- ? { ...m, content, isStreaming: false }+ ? {+ ...m,+ content: text,+ isStreaming: false,+ sources: metadata?.sources,+ suggestedQuestions: metadata?.suggestedQuestions,+ }: m,),);@@ -409,6 +519,11 @@ function ViewerChatPanelContent({}};+ const handleSuggestedQuestion = (question: string) => {+ if (isLoading) return;+ handleSubmit({ text: question, files: [] });+ };+return (<div className="flex h-full w-full flex-col border-l border-gray-200 bg-white">{/* Header with Thread Selector and New Chat Button */}@@ -463,28 +578,73 @@ function ViewerChatPanelContent({<Conversation className="flex-1"><ConversationContent className="gap-4 p-4">{messages.length === 0 ? (- <ConversationEmptyState- icon={- <div className="flex size-12 items-center justify-center rounded-full bg-primary/10">- <SparklesIcon className="size-6 text-primary" />+ <ConversationEmptyState className="items-start text-left">+ <div className="flex w-full flex-col items-start gap-5 px-2">+ <div>+ <div className="flex items-center gap-2">+ <div className="flex size-8 items-center justify-center rounded-full bg-primary/10">+ <SparklesIcon className="size-4 text-primary" />+ </div>+ <h3 className="text-base font-semibold text-gray-900">+ Welcome+ </h3>+ </div>+ <p className="mt-1 text-sm text-gray-500">+ Your AI Data Room Agent is ready to help.+ </p></div>- }- title="Ask me anything"- description={`I can help you understand and analyze the content of this ${dataroomId ? "data room" : "document"}.`}- />++ <div className="w-full space-y-3">+ <div className="flex items-start gap-2.5">+ <SearchIcon className="mt-0.5 size-3.5 shrink-0 text-gray-400" />+ <p className="text-xs leading-relaxed text-gray-500">+ Get answers to questions about your{" "}+ {dataroomId ? "data room" : "document"} files. Open a+ specific document to ask targeted questions.+ </p>+ </div>+ <div className="flex items-start gap-2.5">+ <MessageSquareTextIcon className="mt-0.5 size-3.5 shrink-0 text-gray-400" />+ <p className="text-xs leading-relaxed text-gray-500">+ Ask clear, specific questions for more accurate results.+ Multiple languages and document types are supported.+ </p>+ </div>+ <div className="flex items-start gap-2.5">+ <ShieldCheckIcon className="mt-0.5 size-3.5 shrink-0 text-gray-400" />+ <p className="text-xs leading-relaxed text-gray-500">+ All conversations are private and confidential.+ </p>+ </div>+ </div>+ </div>+ </ConversationEmptyState>) : (<>- {messages.map((message) => (- <div key={message.id} className="space-y-2">- <ChatMessage- role={message.role}- content={message.content}- isStreaming={message.isStreaming}- />- </div>- ))}+ {(() => {+ const lastAssistantIdx = messages.reduce(+ (acc, m, i) => (m.role === "assistant" ? i : acc),+ -1,+ );+ return messages.map((message, idx) => (+ <div key={message.id} className="space-y-2">+ <ChatMessage+ role={message.role}+ content={message.content}+ isStreaming={message.isStreaming}+ sources={message.sources}+ suggestedQuestions={message.suggestedQuestions}+ onSuggestedQuestionClick={handleSuggestedQuestion}+ isLastAssistantMessage={+ message.role === "assistant" &&+ idx === lastAssistantIdx &&+ !isLoading+ }+ />+ </div>+ ));+ })()}- {/* Show loading indicator while waiting for response */}{isLoading && messages[messages.length - 1]?.role === "user" && (<div className="flex items-center gap-2 text-sm text-muted-foreground"><Loader2 className="size-4 animate-spin" />diff --git a/ee/features/ai/lib/chat/get-filtered-dataroom-document-ids.ts b/ee/features/ai/lib/chat/get-filtered-dataroom-document-ids.tsindex 3d32967b..57d78780 100644--- a/ee/features/ai/lib/chat/get-filtered-dataroom-document-ids.ts+++ b/ee/features/ai/lib/chat/get-filtered-dataroom-document-ids.ts@@ -1,19 +1,58 @@+import { ItemType } from "@prisma/client";+import prisma from "@/lib/prisma";/*** Get filtered dataroom document IDs based on link permissions* @param dataroomId - The dataroom ID* @param linkId - The link ID (optional, for external visitors)- * @returns Array of accessible dataroom document IDs+ * @returns Array of accessible dataroom document IDs, or undefined if unrestricted*/export async function getFilteredDataroomDocumentIds(dataroomId: string,linkId?: string,-): Promise<string[]> {+): Promise<string[] | undefined> {+ const getDescendantFolderIds = (+ rootFolderIds: string[],+ folders: { id: string; parentId: string | null }[],+ deniedFolderIds: Set<string>,+ ): Set<string> => {+ const folderIds = new Set(rootFolderIds);+ const queue = [...rootFolderIds];++ // Build parent -> children map for O(1) descendant traversal+ const childFoldersByParent = new Map<string, string[]>();+ for (const folder of folders) {+ if (!folder.parentId) continue;+ const children = childFoldersByParent.get(folder.parentId) ?? [];+ children.push(folder.id);+ childFoldersByParent.set(folder.parentId, children);+ }++ while (queue.length > 0) {+ const currentFolderId = queue.shift();+ if (!currentFolderId) continue;++ const children = childFoldersByParent.get(currentFolderId) ?? [];+ for (const childFolderId of children) {+ // Explicit deny on a folder should block inherited access via parent folder.+ if (deniedFolderIds.has(childFolderId)) {+ continue;+ }++ if (folderIds.has(childFolderId)) continue;+ folderIds.add(childFolderId);+ queue.push(childFolderId);+ }+ }++ return folderIds;+ };+try {- // If external visitor with link, check permission groups+ // No link means this is an internal/team context without link-level restrictionsif (!linkId) {- return [];+ return undefined;}const link = await prisma.link.findUnique({@@ -28,41 +67,132 @@ export async function getFilteredDataroomDocumentIds(return [];}- let accessibleDocuments: { itemId: string }[] = [];+ // If link has no group restrictions, treat as unrestricted dataroom access+ if (!link.groupId && !link.permissionGroupId) {+ return undefined;+ }++ let accessControls: {+ itemId: string;+ itemType: ItemType;+ canView: boolean;+ canDownload: boolean;+ }[] = [];if (link.permissionGroupId) {- accessibleDocuments = await prisma.permissionGroupAccessControls.findMany(- {- where: {- groupId: link.permissionGroupId,- itemType: "DATAROOM_DOCUMENT",- canView: true,- },- select: {- itemId: true,+ accessControls = await prisma.permissionGroupAccessControls.findMany({+ where: {+ groupId: link.permissionGroupId,+ itemType: {+ in: [ItemType.DATAROOM_DOCUMENT, ItemType.DATAROOM_FOLDER],},},- );+ select: {+ itemId: true,+ itemType: true,+ canView: true,+ canDownload: true,+ },+ });}if (link.groupId) {- accessibleDocuments = await prisma.viewerGroupAccessControls.findMany({+ // Legacy group permissions take precedence when present+ accessControls = await prisma.viewerGroupAccessControls.findMany({where: {groupId: link.groupId,- itemType: "DATAROOM_DOCUMENT",- canView: true,+ itemType: {+ in: [ItemType.DATAROOM_DOCUMENT, ItemType.DATAROOM_FOLDER],+ },},select: {itemId: true,+ itemType: true,+ canView: true,+ canDownload: true,},});}- const dataroomDocumentIds = accessibleDocuments.map(- (document) => document.itemId,+ const isAllowed = (control: { canView: boolean; canDownload: boolean }) =>+ control.canView || control.canDownload;++ const explicitlyAllowedDocumentIds = new Set(+ accessControls+ .filter(+ (permission) =>+ permission.itemType === ItemType.DATAROOM_DOCUMENT &&+ isAllowed(permission),+ )+ .map((permission) => permission.itemId),+ );++ const explicitlyDeniedDocumentIds = new Set(+ accessControls+ .filter(+ (permission) =>+ permission.itemType === ItemType.DATAROOM_DOCUMENT &&+ !isAllowed(permission),+ )+ .map((permission) => permission.itemId),+ );++ const allowedFolderIds = accessControls+ .filter(+ (permission) =>+ permission.itemType === ItemType.DATAROOM_FOLDER &&+ isAllowed(permission),+ )+ .map((permission) => permission.itemId);++ const deniedFolderIds = new Set(+ accessControls+ .filter(+ (permission) =>+ permission.itemType === ItemType.DATAROOM_FOLDER &&+ !isAllowed(permission),+ )+ .map((permission) => permission.itemId),);- return dataroomDocumentIds;+ if (allowedFolderIds.length > 0) {+ const folders = await prisma.dataroomFolder.findMany({+ where: { dataroomId },+ select: {+ id: true,+ parentId: true,+ },+ });++ // Start from explicitly allowed folder IDs and include descendants,+ // while respecting explicit folder denies.+ const accessibleFolderIds = Array.from(+ getDescendantFolderIds(allowedFolderIds, folders, deniedFolderIds),+ );++ if (accessibleFolderIds.length > 0) {+ const folderDocuments = await prisma.dataroomDocument.findMany({+ where: {+ dataroomId,+ folderId: { in: accessibleFolderIds },+ },+ select: {+ id: true,+ },+ });++ for (const document of folderDocuments) {+ explicitlyAllowedDocumentIds.add(document.id);+ }+ }+ }++ // Explicit document denies should always override inherited folder access.+ for (const deniedDocumentId of explicitlyDeniedDocumentIds) {+ explicitlyAllowedDocumentIds.delete(deniedDocumentId);+ }++ return Array.from(explicitlyAllowedDocumentIds);} catch (error) {console.error("Error getting filtered file IDs:", error);throw new Error("Failed to get filtered file IDs");diff --git a/ee/features/ai/lib/chat/send-message.ts b/ee/features/ai/lib/chat/send-message.tsindex d01fcfe4..348da2f9 100644--- a/ee/features/ai/lib/chat/send-message.ts+++ b/ee/features/ai/lib/chat/send-message.ts@@ -1,5 +1,5 @@import { OpenAIResponsesProviderOptions, openai } from "@ai-sdk/openai";-import { streamText } from "ai";+import { generateText, streamText } from "ai";import prisma from "@/lib/prisma";@@ -7,11 +7,405 @@ interface SendMessageOptions {chatId: string;content: string;vectorStoreId: string;+ /** Permission-scoped dataroom document IDs; undefined means unrestricted */filteredDataroomDocumentIds?: string[];/** Filter file_search results to a specific document by its ID */filterDocumentId?: string;/** User-selected dataroom document IDs to filter file_search results */userSelectedDataroomDocumentIds?: string[];+ /** Dataroom ID for citation-to-document mapping */+ dataroomId?: string;+ /** Link ID used to generate canonical viewer URLs */+ linkId?: string;+}++interface CitationCandidate {+ fileId?: string;+ filename?: string;+ page?: number;+}++interface ResolvedReference {+ dataroomDocumentId: string;+ documentName: string;+ url: string;+ page?: number;+ folderPath?: string;+}++export const CHAT_METADATA_PREFIX = "\n\n<!-- CHAT_METADATA:";+export const CHAT_METADATA_SUFFIX = " -->";++export interface ChatStreamSource {+ id: string;+ name: string;+ url: string;+ dataroomDocumentId?: string;+ page?: number;+ folderPath?: string;+}++export interface ChatStreamMetadata {+ sources: ChatStreamSource[];+ suggestedQuestions: string[];+}++function normalizePageNumber(value: unknown): number | undefined {+ if (typeof value !== "number" || !Number.isFinite(value)) {+ return undefined;+ }++ const rounded = Math.round(value);+ if (rounded <= 0) {+ // Some providers report first page as 0-indexed.+ return rounded === 0 ? 1 : undefined;+ }++ return rounded;+}++function extractPageNumberFromRecord(+ record: Record<string, unknown>,+): number | undefined {+ const directPage = normalizePageNumber(+ record.page_number ?? record.page ?? record.pageNumber,+ );+ if (directPage !== undefined) {+ return directPage;+ }++ // Some payloads expose 0-based page index.+ const pageIndex = normalizePageNumber(record.page_index ?? record.pageIndex);+ if (pageIndex !== undefined) {+ return pageIndex + 1;+ }++ const location =+ typeof record.location === "object" && record.location+ ? (record.location as Record<string, unknown>)+ : null;+ if (location) {+ const locationPage = normalizePageNumber(+ location.page_number ?? location.page ?? location.pageNumber,+ );+ if (locationPage !== undefined) {+ return locationPage;+ }++ const locationPageIndex = normalizePageNumber(+ location.page_index ?? location.pageIndex,+ );+ if (locationPageIndex !== undefined) {+ return locationPageIndex + 1;+ }+ }++ return undefined;+}++function extractFileCitations(response: unknown): CitationCandidate[] {+ const citations: CitationCandidate[] = [];+ const visited = new Set<object>();++ const addCitation = (candidate: CitationCandidate) => {+ if (!candidate.fileId && !candidate.filename) return;+ citations.push(candidate);+ };++ const visit = (node: unknown) => {+ if (!node || typeof node !== "object") return;++ if (visited.has(node as object)) {+ return;+ }+ visited.add(node as object);++ const record = node as Record<string, unknown>;++ // OpenAI often provides annotations with type=file_citation.+ if (record.type === "file_citation") {+ addCitation({+ fileId:+ (record.file_id as string | undefined) ||+ (record.fileId as string | undefined),+ filename:+ (record.filename as string | undefined) ||+ (record.file_name as string | undefined),+ page: extractPageNumberFromRecord(record),+ });+ }++ // Sometimes payloads are nested under file_citation key.+ if (record.file_citation && typeof record.file_citation === "object") {+ const nested = record.file_citation as Record<string, unknown>;+ addCitation({+ fileId:+ (nested.file_id as string | undefined) ||+ (nested.fileId as string | undefined),+ filename:+ (nested.filename as string | undefined) ||+ (nested.file_name as string | undefined),+ page: extractPageNumberFromRecord(nested),+ });+ visit(record.file_citation);+ }++ for (const value of Object.values(record)) {+ if (Array.isArray(value)) {+ for (const item of value) {+ visit(item);+ }+ } else {+ visit(value);+ }+ }+ };++ visit(response);++ return citations;+}++function stripTrailingReferencesSection(text: string): string {+ return text+ .replace(+ /\n{2,}(?:#{1,6}\s*)?References\s*\n(?:\s*\n)?(?:[-*]\s.*(?:\n|$))+$/i,+ "",+ )+ .trimEnd();+}++function escapeMarkdownLinkText(value: string): string {+ return value.replace(/\[/g, "\\[").replace(/\]/g, "\\]");+}++async function resolveReferencesFromCitations({+ dataroomId,+ linkId,+ citations,+}: {+ dataroomId?: string;+ linkId?: string;+ citations: CitationCandidate[];+}): Promise<ResolvedReference[]> {+ if (!dataroomId || !linkId || citations.length === 0) {+ return [];+ }++ const citedFileIds = Array.from(+ new Set(+ citations+ .map((citation) => citation.fileId)+ .filter((fileId): fileId is string => Boolean(fileId)),+ ),+ );++ const fileIdToDocument = new Map<+ string,+ { dataroomDocumentId: string; documentName: string; folderPath?: string }+ >();++ if (citedFileIds.length > 0) {+ const dataroomDocuments = await prisma.dataroomDocument.findMany({+ where: {+ dataroomId,+ document: {+ versions: {+ some: {+ isPrimary: true,+ fileId: { in: citedFileIds },+ },+ },+ },+ },+ select: {+ id: true,+ folder: {+ select: {+ name: true,+ path: true,+ },+ },+ document: {+ select: {+ name: true,+ versions: {+ where: { isPrimary: true },+ take: 1,+ select: {+ fileId: true,+ },+ },+ },+ },+ },+ });++ for (const dataroomDocument of dataroomDocuments) {+ const primaryFileId = dataroomDocument.document.versions[0]?.fileId;+ if (!primaryFileId) continue;+ fileIdToDocument.set(primaryFileId, {+ dataroomDocumentId: dataroomDocument.id,+ documentName: dataroomDocument.document.name,+ folderPath: dataroomDocument.folder?.path,+ });+ }+ }++ const citedFilenames = Array.from(+ new Set(+ citations+ .map((citation) => citation.filename)+ .filter((filename): filename is string => Boolean(filename)),+ ),+ );++ const uniqueFilenameToDocument = new Map<+ string,+ { dataroomDocumentId: string; documentName: string; folderPath?: string }+ >();++ if (citedFilenames.length > 0) {+ const namedDataroomDocuments = await prisma.dataroomDocument.findMany({+ where: {+ dataroomId,+ document: {+ name: { in: citedFilenames },+ },+ },+ select: {+ id: true,+ folder: {+ select: {+ name: true,+ path: true,+ },+ },+ document: {+ select: {+ name: true,+ },+ },+ },+ });++ const groupedByName = new Map<+ string,+ { dataroomDocumentId: string; documentName: string; folderPath?: string }[]+ >();+ for (const item of namedDataroomDocuments) {+ const current = groupedByName.get(item.document.name) ?? [];+ current.push({+ dataroomDocumentId: item.id,+ documentName: item.document.name,+ folderPath: item.folder?.path,+ });+ groupedByName.set(item.document.name, current);+ }++ for (const [filename, docs] of groupedByName.entries()) {+ if (docs.length === 1) {+ uniqueFilenameToDocument.set(filename, docs[0]);+ }+ }+ }++ const references: ResolvedReference[] = [];+ const seen = new Set<string>();++ for (const citation of citations) {+ const mappedByFileId = citation.fileId+ ? fileIdToDocument.get(citation.fileId)+ : undefined;+ const mappedByFilename = citation.filename+ ? uniqueFilenameToDocument.get(citation.filename)+ : undefined;++ const mapped = mappedByFileId ?? mappedByFilename;+ if (!mapped) continue;++ const dedupeKey = `${mapped.dataroomDocumentId}:${citation.page ?? ""}`;+ if (seen.has(dedupeKey)) continue;+ seen.add(dedupeKey);++ references.push({+ dataroomDocumentId: mapped.dataroomDocumentId,+ documentName: mapped.documentName,+ url: `/view/${linkId}/d/${mapped.dataroomDocumentId}${+ citation.page ? `#page=${citation.page}` : ""+ }`,+ page: citation.page,+ folderPath: mapped.folderPath,+ });+ }++ return references;+}++function buildReferencesSection(references: ResolvedReference[]): string {+ const lines = ["", "", "References", ""];++ if (references.length === 0) {+ lines.push("- None");+ return lines.join("\n");+ }++ for (const reference of references) {+ const documentLabel = escapeMarkdownLinkText(reference.documentName);+ const pageSuffix = reference.page ? ` - p. ${reference.page}` : "";+ lines.push(`- [${documentLabel}](${reference.url})${pageSuffix}`);+ }++ return lines.join("\n");+}++async function generateSuggestedQuestions(+ userQuestion: string,+ assistantResponse: string,+): Promise<string[]> {+ try {+ const { text } = await generateText({+ model: openai("gpt-4o-mini"),+ maxOutputTokens: 200,+ messages: [+ {+ role: "system",+ content:+ "Generate exactly 3 concise follow-up questions a user might ask based on this conversation. Output only the questions, one per line. No numbering, no bullets, no prefixes.",+ },+ {+ role: "user",+ content: `User asked: "${userQuestion}"\n\nAssistant answered: "${assistantResponse.slice(0, 500)}"`,+ },+ ],+ });++ return text+ .split("\n")+ .map((q) => q.trim())+ .filter((q) => q.length > 0 && q.endsWith("?"))+ .slice(0, 3);+ } catch {+ return [];+ }+}++function buildStreamMetadataTail(+ references: ResolvedReference[],+ suggestedQuestions: string[],+): string {+ const metadata: ChatStreamMetadata = {+ sources: references.map((ref, idx) => ({+ id: `D-${idx + 1}`,+ name: ref.documentName,+ url: ref.url,+ dataroomDocumentId: ref.dataroomDocumentId,+ page: ref.page,+ folderPath: ref.folderPath,+ })),+ suggestedQuestions,+ };+ return `${CHAT_METADATA_PREFIX}${JSON.stringify(metadata)}${CHAT_METADATA_SUFFIX}`;}/**@@ -25,6 +419,8 @@ export async function sendMessage({filteredDataroomDocumentIds,filterDocumentId,userSelectedDataroomDocumentIds,+ dataroomId,+ linkId,}: SendMessageOptions) {// Get conversation history from databaseconst history = await prisma.chatMessage.findMany({@@ -33,11 +429,20 @@ export async function sendMessage({take: 10,});+ const hasPriorAssistantReply = history.some((m) => m.role === "assistant");+ const reasoningEffort = "minimal";+ const maxOutputTokens = hasPriorAssistantReply ? 420 : 220;+// Save user messageawait prisma.chatMessage.create({data: { chatId, role: "user", content },});+ let resolveReferencesForStream: (value: string) => void = () => {};+ const referencesForStream = new Promise<string>((resolve) => {+ resolveReferencesForStream = resolve;+ });+// Build messages for AI SDKconst messages = [{@@ -45,11 +450,30 @@ export async function sendMessage({content: `You are a helpful AI assistant answering questions about documents.Use the file_search tool to find relevant information from the documents.Always cite sources with document names and page numbers when providing information.+Do not include a "References" section in your answer. References are appended automatically by the system.+Keep answers concise, direct, and non-repetitive.+Start with the direct answer in the first sentence.+Default format: 1 short paragraph or up to 2 bullets.+Avoid long document lists unless the user explicitly asks for a full list.+Only mention documents that are directly needed for the user's question.+${+ !hasPriorAssistantReply+ ? `This is the first assistant reply in the conversation.+Prioritize speed:+- Start with a direct answer in 1-2 sentences.+- Add at most 2 short bullets only if needed.+- Keep it under 80 words before references.+If deeper analysis might help, add one short sentence offering a deeper follow-up.`+ : ""+}If you cannot find the answer in the documents, say so clearly.`,},...history.map((m) => ({role: m.role as "user" | "assistant",- content: m.content,+ content:+ m.role === "assistant"+ ? stripTrailingReferencesSection(m.content)+ : m.content,})),{ role: "user" as const, content },];@@ -59,6 +483,9 @@ If you cannot find the answer in the documents, say so clearly.`,vectorStoreIds: [vectorStoreId],};+ const NO_ACCESS_SENTINEL = "__no_access__";+ let effectiveDataroomDocumentIds: string[] | undefined;+// Add document filter when viewing a specific document in a dataroomif (filterDocumentId) {fileSearchOptions.filters = {@@ -73,35 +500,39 @@ If you cannot find the answer in the documents, say so clearly.`,// User explicitly selected documents to filter to// Intersect with permission-based filter if it existslet effectiveIds = userSelectedDataroomDocumentIds;- if (filteredDataroomDocumentIds && filteredDataroomDocumentIds.length > 0) {+ if (filteredDataroomDocumentIds !== undefined) {// Only include user-selected IDs that are also in the permission-filtered listeffectiveIds = userSelectedDataroomDocumentIds.filter((id) =>filteredDataroomDocumentIds.includes(id),);// Security: If intersection is empty (user sent unauthorized IDs),- // fall back to the permission-based filter to prevent searching all documents+ // fall back to the permission-based filterif (effectiveIds.length === 0) {effectiveIds = filteredDataroomDocumentIds;}}- if (effectiveIds.length > 0) {+ effectiveDataroomDocumentIds = effectiveIds;+ } else if (filteredDataroomDocumentIds !== undefined) {+ // Permission-restricted dataroom chat scope+ effectiveDataroomDocumentIds = filteredDataroomDocumentIds;+ }++ if (!filterDocumentId && effectiveDataroomDocumentIds !== undefined) {+ if (effectiveDataroomDocumentIds.length > 0) {fileSearchOptions.filters = {type: "in",key: "dataroomDocumentId",- value: effectiveIds,+ value: effectiveDataroomDocumentIds,+ };+ } else {+ // Keep retrieval restricted when the viewer has no accessible documents+ // by applying a guaranteed-empty metadata filter.+ fileSearchOptions.filters = {+ type: "eq",+ key: "dataroomDocumentId",+ value: NO_ACCESS_SENTINEL,};}- } else if (- filteredDataroomDocumentIds &&- filteredDataroomDocumentIds.length > 0- ) {- // Only apply filter if there are specific documents to filter to- // An empty array would filter out ALL documents- fileSearchOptions.filters = {- type: "in",- key: "dataroomDocumentId",- value: filteredDataroomDocumentIds,- };}const latestMessage = history.at(0);@@ -112,6 +543,7 @@ If you cannot find the answer in the documents, say so clearly.`,// Use AI SDK streamText with file_search toolconst result = streamText({model: openai.responses("gpt-4o-mini"),+ maxOutputTokens,messages,tools: {file_search: openai.tools.fileSearch(fileSearchOptions),@@ -119,31 +551,110 @@ If you cannot find the answer in the documents, say so clearly.`,providerOptions: {openai: {previousResponseId,- } satisfies OpenAIResponsesProviderOptions,+ } as OpenAIResponsesProviderOptions,},onFinish: async ({ text, usage, response }) => {- await Promise.all([- prisma.chatMessage.create({- data: {- chatId,- role: "assistant",- content: text,- metadata: {- usage,- vectorStoreId,- responseId: response.id,- modelId: response.modelId,- filters: fileSearchOptions.filters,+ try {+ const shouldAppendReferences = Boolean(dataroomId && linkId);+ const citations = extractFileCitations(response);++ const [resolvedReferences, suggestedQuestions] = await Promise.all([+ resolveReferencesFromCitations({+ dataroomId,+ linkId,+ citations,+ }),+ generateSuggestedQuestions(content, text),+ ]);++ const maxReferences = hasPriorAssistantReply ? 6 : 3;+ const limitedReferences = resolvedReferences.slice(0, maxReferences);++ const referencesSection = shouldAppendReferences+ ? buildReferencesSection(limitedReferences)+ : "";++ const strippedText = stripTrailingReferencesSection(text).trim();+ const rawTrimmedText = text.trim();+ const hasRawText = rawTrimmedText.length > 0;+ const resolvedText =+ strippedText ||+ rawTrimmedText ||+ "I couldn't find that in the indexed documents.";++ const metadataTail = buildStreamMetadataTail(+ shouldAppendReferences ? limitedReferences : [],+ suggestedQuestions,+ );++ const streamTail = hasRawText+ ? metadataTail+ : `${resolvedText}${metadataTail}`;++ resolveReferencesForStream(streamTail);++ const finalContent = shouldAppendReferences+ ? `${resolvedText}${referencesSection}`+ : resolvedText;++ await Promise.all([+ prisma.chatMessage.create({+ data: {+ chatId,+ role: "assistant",+ content: finalContent,+ metadata: {+ usage: usage as any,+ vectorStoreId,+ responseId: response.id,+ modelId: response.modelId,+ filters: fileSearchOptions.filters as any,+ ...(shouldAppendReferences && {+ citations: citations as any,+ references: limitedReferences as any,+ }),+ suggestedQuestions,+ reasoningEffort,+ },},- },- }),- prisma.chat.update({- where: { id: chatId },- data: { lastMessageAt: new Date() },- }),- ]);+ }),+ prisma.chat.update({+ where: { id: chatId },+ data: { lastMessageAt: new Date() },+ }),+ ]);+ } catch (error) {+ resolveReferencesForStream("");+ console.error("Error finalizing AI response references:", error);++ await Promise.all([+ prisma.chatMessage.create({+ data: {+ chatId,+ role: "assistant",+ content: text,+ metadata: {+ usage,+ vectorStoreId,+ responseId: response.id,+ modelId: response.modelId,+ filters: fileSearchOptions.filters,+ referenceError:+ error instanceof Error ? error.message : "Unknown error",+ },+ },+ }),+ prisma.chat.update({+ where: { id: chatId },+ data: { lastMessageAt: new Date() },+ }),+ ]);+ }},});- return result;+ return {+ result,+ referencesForStream,+ };}