RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(web): make audio attachments and references playable (#19842)
euvre947975d23137
diff --git a/web/src/components/document-preview/audio-preview.test.tsx b/web/src/components/document-preview/audio-preview.test.tsxnew file mode 100644index 0000000000..ce4befe712--- /dev/null+++ b/web/src/components/document-preview/audio-preview.test.tsx@@ -0,0 +1,100 @@+import request from '@/utils/request';+import { render, screen, waitFor } from '@testing-library/react';+import { AudioPreviewer } from './audio-preview';++jest.mock('@/utils/request', () => ({+ __esModule: true,+ default: jest.fn(),+}));++jest.mock('@/components/ui/spin', () => ({+ Spin: () => <div data-testid="spin" />,+}));++const MockRequest = jest.mocked(request);++beforeAll(() => {+ URL.createObjectURL = jest.fn(+ (blob: Blob | MediaSource) => `blob:${(blob as Blob).type}`,+ );+ URL.revokeObjectURL = jest.fn();+});++afterAll(() => {+ jest.restoreAllMocks();+});++beforeEach(() => {+ jest.clearAllMocks();+});++describe('AudioPreviewer', () => {+ it('fetches the audio through the authenticated url and renders a player', async () => {+ MockRequest.mockResolvedValue({+ data: new Blob([new Uint8Array([1, 2, 3])], { type: 'audio/mpeg' }),+ });++ render(<AudioPreviewer url="/api/v1/agents/attachments/doc1/preview" />);++ await waitFor(() =>+ expect(screen.getByTestId('document-audio-player')).toBeInTheDocument(),+ );+ expect(MockRequest).toHaveBeenCalledWith(+ '/api/v1/agents/attachments/doc1/preview',+ { method: 'GET', responseType: 'blob' },+ );+ expect(+ screen.getByTestId('document-audio-player').getAttribute('src'),+ ).toBe('blob:audio/mpeg');+ });++ it('renders an error state and no player when the response carries no blob', async () => {+ MockRequest.mockResolvedValue({ data: { code: 1999 } });++ render(<AudioPreviewer url="/api/v1/agents/attachments/doc2/preview" />);++ await waitFor(() =>+ expect(screen.getByTestId('document-audio-error')).toBeInTheDocument(),+ );+ expect(+ screen.queryByTestId('document-audio-player'),+ ).not.toBeInTheDocument();+ expect(screen.queryByTestId('spin')).not.toBeInTheDocument();+ expect(URL.createObjectURL).not.toHaveBeenCalled();+ });++ it('ignores a stale response when the url changes mid-flight', async () => {+ const pending: Record<string, (value: { data: Blob }) => void> = {};+ MockRequest.mockImplementation(+ (url: unknown) =>+ new Promise((resolve) => {+ pending[url as string] = resolve;+ }),+ );++ const oldUrl = '/api/v1/agents/attachments/old/preview';+ const newUrl = '/api/v1/agents/attachments/new/preview';+ const { rerender } = render(<AudioPreviewer url={oldUrl} />);+ rerender(<AudioPreviewer url={newUrl} />);++ pending[oldUrl]({+ data: new Blob([new Uint8Array([1])], { type: 'audio/old' }),+ });++ expect(+ screen.queryByTestId('document-audio-player'),+ ).not.toBeInTheDocument();+ expect(URL.createObjectURL).not.toHaveBeenCalled();++ pending[newUrl]({+ data: new Blob([new Uint8Array([2])], { type: 'audio/new' }),+ });++ await waitFor(() =>+ expect(screen.getByTestId('document-audio-player')).toBeInTheDocument(),+ );+ expect(+ screen.getByTestId('document-audio-player').getAttribute('src'),+ ).toBe('blob:audio/new');+ });+});diff --git a/web/src/components/document-preview/audio-preview.tsx b/web/src/components/document-preview/audio-preview.tsxnew file mode 100644index 0000000000..42e6c094c0--- /dev/null+++ b/web/src/components/document-preview/audio-preview.tsx@@ -0,0 +1,101 @@+import { Spin } from '@/components/ui/spin';+import request from '@/utils/request';+import classNames from 'classnames';+import { useEffect, useState } from 'react';++interface AudioPreviewerProps {+ className?: string;+ url: string;+}++export const AudioPreviewer: React.FC<AudioPreviewerProps> = ({+ className,+ url,+}) => {+ const [audioSrc, setAudioSrc] = useState<string | null>(null);+ const [isLoading, setIsLoading] = useState<boolean>(true);+ const [loadFailed, setLoadFailed] = useState<boolean>(false);++ useEffect(() => {+ if (!url) {+ return;+ }+ let stale = false;+ setIsLoading(true);+ setLoadFailed(false);++ const loadAudio = async () => {+ try {+ const res = await request(url, {+ method: 'GET',+ responseType: 'blob',+ });+ if (stale) {+ return;+ }+ if (!(res.data instanceof Blob)) {+ setLoadFailed(true);+ return;+ }+ setAudioSrc(URL.createObjectURL(res.data));+ } catch {+ if (!stale) {+ setLoadFailed(true);+ }+ } finally {+ if (!stale) {+ setIsLoading(false);+ }+ }+ };++ loadAudio();++ return () => {+ stale = true;+ };+ }, [url]);++ useEffect(() => {+ return () => {+ if (audioSrc) {+ URL.revokeObjectURL(audioSrc);+ }+ };+ }, [audioSrc]);++ return (+ <div+ className={classNames(+ 'relative w-full h-full p-4 bg-background-paper border border-border-normal rounded-md audio-previewer',+ className,+ )}+ >+ {isLoading && (+ <div className="absolute inset-0 flex items-center justify-center">+ <Spin />+ </div>+ )}++ {!isLoading && audioSrc && (+ <div className="flex h-full items-center justify-center">+ <audio+ src={audioSrc}+ controls+ className="w-full max-w-2xl"+ data-testid="document-audio-player"+ />+ </div>+ )}++ {!isLoading && !audioSrc && loadFailed && (+ <div+ className="flex h-full items-center justify-center text-text-secondary"+ data-testid="document-audio-error"+ >+ Failed to load audio+ </div>+ )}+ </div>+ );+};diff --git a/web/src/components/document-preview/index.tsx b/web/src/components/document-preview/index.tsxindex 4e1637390c..b990e2f523 100644--- a/web/src/components/document-preview/index.tsx+++ b/web/src/components/document-preview/index.tsx@@ -16,7 +16,8 @@import { memo } from 'react';-import { Images } from '@/constants/common';+import { AudioExtensions, Images } from '@/constants/common';+import { AudioPreviewer } from './audio-preview';import CSVFileViewer from './csv-preview';import { DocPreviewer } from './doc-preview';import { EpubPreviewer } from './epub-preview';@@ -88,6 +89,11 @@ const DocumentPreview = function ({<VideoPreviewer className={className} url={url} /></section>)}+ {AudioExtensions.indexOf(fileType) > -1 && (+ <section>+ <AudioPreviewer className={className} url={url} />+ </section>+ )}{['ppt', 'pptx'].indexOf(fileType) > -1 && (<section><PptPreviewer className={className} url={url} />diff --git a/web/src/components/file-icon/index.tsx b/web/src/components/file-icon/index.tsxindex b113df10da..ece51eb3cb 100644--- a/web/src/components/file-icon/index.tsx+++ b/web/src/components/file-icon/index.tsx@@ -34,7 +34,7 @@ const FileIcon = ({ name, id }: IProps) => {const { data: fileThumbnails, setDocumentIds } =useFetchDocumentThumbnailsByIds();const fileThumbnail = fileThumbnails[id];- const blobUrl = useAuthenticatedImageUrl(fileThumbnail);+ const { src: blobUrl } = useAuthenticatedImageUrl(fileThumbnail);useEffect(() => {if (id) {diff --git a/web/src/components/image/index.tsx b/web/src/components/image/index.tsxindex 0a01c7a567..e87ec9f41b 100644--- a/web/src/components/image/index.tsx+++ b/web/src/components/image/index.tsx@@ -167,33 +167,42 @@ export const useDocumentImageUrl = (return imageUrl;};+export type AuthenticatedImageUrlStatus = 'loading' | 'ready' | 'error';+/*** Hook to convert any authenticated URL to a blob URL for use in <img> tags.* Use this for thumbnail URLs or any other API URLs that require authentication.+ * The status distinguishes an in-flight fetch from a failed one, which the+ * empty src alone cannot express.*/-export const useAuthenticatedImageUrl = (url: string | undefined | null) => {- const [imageUrl, setImageUrl] = useState<string>('');+export const useAuthenticatedImageUrl = (+ url: string | undefined | null,+): { src: string; status: AuthenticatedImageUrlStatus } => {+ const [state, setState] = useState<{+ src: string;+ status: AuthenticatedImageUrlStatus;+ }>({ src: '', status: 'loading' });useEffect(() => {if (!url || !isAuthRequiredUrl(url)) {- setImageUrl(url || '');+ setState({ src: url || '', status: 'ready' });return;}const authorization = getAuthorization();let cancelled = false;- setImageUrl('');+ setState({ src: '', status: 'loading' });const { promise, release } = fetchDocumentImage(url, authorization);promise.then((blobUrl) => {if (!cancelled) {- setImageUrl(blobUrl);+ setState({ src: blobUrl, status: 'ready' });}}).catch(() => {if (!cancelled) {- setImageUrl('');+ setState({ src: '', status: 'error' });}});@@ -203,7 +212,7 @@ export const useAuthenticatedImageUrl = (url: string | undefined | null) => {};}, [url]);- return imageUrl;+ return state;};/**@@ -219,7 +228,7 @@ export const AuthenticatedImg = ({}: React.ImgHTMLAttributes<HTMLImageElement> & {fallback?: React.ReactNode;}) => {- const authenticatedSrc = useAuthenticatedImageUrl(src);+ const { src: authenticatedSrc } = useAuthenticatedImageUrl(src);if (!authenticatedSrc) return fallback ?? null;diff --git a/web/src/components/next-message-item/uploaded-message-files.test.tsx b/web/src/components/next-message-item/uploaded-message-files.test.tsxnew file mode 100644index 0000000000..ceb15272f2--- /dev/null+++ b/web/src/components/next-message-item/uploaded-message-files.test.tsx@@ -0,0 +1,177 @@+import { fireEvent, render, screen } from '@testing-library/react';+import { UploadedMessageFiles } from './uploaded-message-files';+import { isAudioFile } from './utils';++jest.mock('../file-icon', () => ({+ __esModule: true,+ default: ({ name }: { name: string }) => (+ <div data-testid="file-icon">{name}</div>+ ),+}));++const mockUseAuthenticatedImageUrl = jest.fn(+ (+ url: string | null | undefined,+ ): {+ src: string;+ status: 'loading' | 'ready' | 'error';+ } => ({+ src: url ?? '',+ status: 'ready',+ }),+);++jest.mock('../image', () => ({+ __esModule: true,+ useAuthenticatedImageUrl: (url: string | null | undefined) =>+ mockUseAuthenticatedImageUrl(url),+}));++jest.mock('../svg-icon', () => ({+ __esModule: true,+ default: ({ name }: { name: string }) => (+ <div data-testid="svg-icon">{name}</div>+ ),+}));++jest.mock('../ui/modal/modal', () => ({+ Modal: ({ open, title, children }: any) =>+ open ? (+ <div data-testid="audio-modal">+ <div data-testid="audio-modal-title">{title}</div>+ {children}+ </div>+ ) : null,+}));++jest.mock('../ui/spin', () => ({+ Spin: () => <div data-testid="spin" />,+}));++jest.mock('react-photo-view', () => ({+ PhotoProvider: ({ children }: any) => <>{children}</>,+ PhotoView: ({ children }: any) => <>{children}</>,+}));++const uploadedAudio = {+ created_at: 1789717195,+ created_by: 'tenant-1',+ extension: 'mp3',+ id: 'attach-1',+ mime_type: 'audio/mpeg',+ name: '1.01a demo reading.mp3',+ preview_url: null,+ size: 1024,+};++beforeAll(() => {+ URL.createObjectURL = jest.fn(() => 'blob:local-audio');+ URL.revokeObjectURL = jest.fn();+});++describe('UploadedMessageFiles audio playback', () => {+ it('detects audio files by mime type and by extension', () => {+ expect(isAudioFile(uploadedAudio)).toBe(true);+ expect(+ isAudioFile({+ ...uploadedAudio,+ mime_type: '',+ name: 'lesson.wav',+ }),+ ).toBe(true);+ expect(+ isAudioFile({+ ...uploadedAudio,+ mime_type: 'application/pdf',+ name: 'lesson.pdf',+ }),+ ).toBe(false);+ });++ it('opens an inline player when an uploaded audio chip is clicked', () => {+ render(+ <UploadedMessageFiles files={[uploadedAudio]}></UploadedMessageFiles>,+ );++ expect(screen.queryByTestId('audio-modal')).not.toBeInTheDocument();++ fireEvent.click(+ screen.getByRole('button', { name: '1.01a demo reading.mp3' }),+ );++ const player = screen.getByTestId('uploaded-audio-player');+ expect(player).toBeInTheDocument();+ expect(player.getAttribute('controls')).not.toBeNull();+ expect(player.getAttribute('src')).toContain(+ '/api/v1/agents/attachments/attach-1/preview',+ );+ });++ it('opens an inline player for an unsent local audio file', () => {+ const localFile = new File([new Uint8Array([1])], 'note.mp3', {+ type: 'audio/mpeg',+ });++ render(<UploadedMessageFiles files={[localFile]}></UploadedMessageFiles>);++ fireEvent.click(screen.getByRole('button', { name: 'note.mp3' }));++ const player = screen.getByTestId('uploaded-audio-player');+ expect(player.getAttribute('src')).toBe('blob:local-audio');+ });++ it('does not open a player for image attachments', () => {+ const image = {+ ...uploadedAudio,+ mime_type: 'image/png',+ name: 'photo.png',+ };++ render(<UploadedMessageFiles files={[image]}></UploadedMessageFiles>);++ fireEvent.click(screen.getByText('photo.png'));++ expect(screen.queryByTestId('audio-modal')).not.toBeInTheDocument();+ });++ it('keeps the spinner while the remote audio preview is loading', () => {+ mockUseAuthenticatedImageUrl.mockReturnValueOnce({+ src: '',+ status: 'loading',+ });++ render(+ <UploadedMessageFiles files={[uploadedAudio]}></UploadedMessageFiles>,+ );++ fireEvent.click(+ screen.getByRole('button', { name: '1.01a demo reading.mp3' }),+ );++ expect(screen.getByTestId('spin')).toBeInTheDocument();+ expect(+ screen.queryByTestId('uploaded-audio-player'),+ ).not.toBeInTheDocument();+ });++ it('shows an error state instead of a spinner when the remote audio fetch fails', () => {+ mockUseAuthenticatedImageUrl.mockReturnValueOnce({+ src: '',+ status: 'error',+ });++ render(+ <UploadedMessageFiles files={[uploadedAudio]}></UploadedMessageFiles>,+ );++ fireEvent.click(+ screen.getByRole('button', { name: '1.01a demo reading.mp3' }),+ );++ expect(screen.getByTestId('uploaded-audio-error')).toBeInTheDocument();+ expect(+ screen.queryByTestId('uploaded-audio-player'),+ ).not.toBeInTheDocument();+ expect(screen.queryByTestId('spin')).not.toBeInTheDocument();+ });+});diff --git a/web/src/components/next-message-item/uploaded-message-files.tsx b/web/src/components/next-message-item/uploaded-message-files.tsxindex 0dea8e5799..24dae8d313 100644--- a/web/src/components/next-message-item/uploaded-message-files.tsx+++ b/web/src/components/next-message-item/uploaded-message-files.tsx@@ -19,12 +19,15 @@ import { IDocumentInfo } from '@/interfaces/database/document';import api from '@/utils/api';import { getExtension } from '@/utils/document-util';import { formatBytes } from '@/utils/file-util';+import { cn } from '@/lib/utils';import { memo, useEffect, useState } from 'react';import { PhotoProvider, PhotoView } from 'react-photo-view';import FileIcon from '../file-icon';import { useAuthenticatedImageUrl } from '../image';+import { Modal } from '../ui/modal/modal';+import { Spin } from '../ui/spin';import SvgIcon from '../svg-icon';-import { getFileMimeType, isImageFile } from './utils';+import { getFileMimeType, isAudioFile, isImageFile } from './utils';interface IProps {files?: File[] | IDocumentInfo[] | UploadResponseDataType[];@@ -43,7 +46,7 @@ function UploadedFileImage({name: string;mimeType?: string;}) {- const src = useAuthenticatedImageUrl(+ const { src } = useAuthenticatedImageUrl(id? api.getAttachmentFilePreview({ docId: id, filename: name, mimeType }): null,@@ -89,6 +92,79 @@ function LocalFileImage({ file, name }: { file: File; name: string }) {);}+// Audio attachments have no visual content, so the chip opens an inline+// player instead of a lightbox: chat uploads live in the per-user downloads+// bucket and are fetched through the authenticated attachment preview+// endpoint, while unsent local files play straight from an object URL.+function AudioPlayerModal({+ file,+ onClose,+}: {+ file: File | UploadResponseDataType | IDocumentInfo;+ onClose: () => void;+}) {+ const name = file.name;+ const isLocal = file instanceof File;+ const { src: remoteSrc, status: remoteStatus } = useAuthenticatedImageUrl(+ !isLocal && file.id+ ? api.getAttachmentFilePreview({+ docId: file.id,+ filename: name,+ mimeType: getFileMimeType(file),+ })+ : null,+ );+ const [localSrc, setLocalSrc] = useState('');++ useEffect(() => {+ if (isLocal) {+ const url = URL.createObjectURL(file);+ setLocalSrc(url);+ return () => URL.revokeObjectURL(url);+ }+ }, [isLocal, file]);++ const src = isLocal ? localSrc : remoteSrc;++ return (+ <Modal+ open+ onCancel={onClose}+ showfooter={false}+ title={+ <div className="flex items-center gap-2">+ {isLocal ? (+ <SvgIcon name={`file-icon/${getExtension(name)}`} width={24} />+ ) : (+ <FileIcon id={file.id ?? ''} name={name}></FileIcon>+ )}+ <span className="truncate">{name}</span>+ </div>+ }+ >+ {src ? (+ <audio+ controls+ src={src}+ className="w-full"+ data-testid="uploaded-audio-player"+ />+ ) : remoteStatus === 'error' ? (+ <div+ className="flex items-center justify-center py-6 text-text-secondary"+ data-testid="uploaded-audio-error"+ >+ Failed to load audio+ </div>+ ) : (+ <div className="flex items-center justify-center py-6">+ <Spin />+ </div>+ )}+ </Modal>+ );+}+type NameWidgetType = {name: string;size: number;@@ -109,6 +185,10 @@ function NameWidget({ name, size }: NameWidgetType) {);}export function InnerUploadedMessageFiles({ files = [] }: IProps) {+ const [playingFile, setPlayingFile] = useState<+ File | UploadResponseDataType | IDocumentInfo | null+ >(null);+return (<PhotoProvider><section className="flex gap-2 pt-2 flex-wrap">@@ -116,9 +196,30 @@ export function InnerUploadedMessageFiles({ files = [] }: IProps) {const name = file.name;const isFile = file instanceof File;const isImage = isImageFile(file);+ const isAudio = isAudioFile(file);return (- <div key={idx} className="flex gap-1 border rounded-md p-1.5">+ <div+ key={idx}+ className={cn(+ 'flex gap-1 border rounded-md p-1.5',+ isAudio && 'cursor-pointer hover:border-accent-primary',+ )}+ role={isAudio ? 'button' : undefined}+ tabIndex={isAudio ? 0 : undefined}+ aria-label={isAudio ? name : undefined}+ onClick={isAudio ? () => setPlayingFile(file) : undefined}+ onKeyDown={+ isAudio+ ? (e) => {+ if (e.key === 'Enter' || e.key === ' ') {+ e.preventDefault();+ setPlayingFile(file);+ }+ }+ : undefined+ }+ >{isImage ? (isFile ? (<LocalFileImage file={file} name={name}></LocalFileImage>@@ -146,6 +247,12 @@ export function InnerUploadedMessageFiles({ files = [] }: IProps) {);})}</section>+ {playingFile && (+ <AudioPlayerModal+ file={playingFile}+ onClose={() => setPlayingFile(null)}+ ></AudioPlayerModal>+ )}</PhotoProvider>);}diff --git a/web/src/components/next-message-item/utils.ts b/web/src/components/next-message-item/utils.tsindex f8d637b01d..28273eb802 100644--- a/web/src/components/next-message-item/utils.ts+++ b/web/src/components/next-message-item/utils.ts@@ -14,6 +14,7 @@* limitations under the License.*/+import { AudioExtensions } from '@/constants/common';import { UploadResponseDataType } from '@/interfaces/database/chat';import { IDocumentInfo } from '@/interfaces/database/document';import { currentReg, parseCitationIndex } from '@/utils/chat';@@ -67,3 +68,13 @@ export function isImageFile(}return ImageExtensions.includes(getExtension(file.name));}++export function isAudioFile(+ file: File | IDocumentInfo | UploadResponseDataType,+): boolean {+ const mimeType = getFileMimeType(file);+ if (mimeType) {+ return mimeType.startsWith('audio/');+ }+ return AudioExtensions.includes(getExtension(file.name));+}diff --git a/web/src/constants/common.ts b/web/src/constants/common.tsindex 88bbb231af..62bc349ec3 100644--- a/web/src/constants/common.ts+++ b/web/src/constants/common.ts@@ -164,6 +164,25 @@ export const Images = ['ico',];+export const AudioExtensions = [+ 'mp3',+ 'wav',+ 'wave',+ 'aac',+ 'flac',+ 'ogg',+ 'oga',+ 'm4a',+ 'wma',+ 'aiff',+ 'aif',+ 'au',+ 'midi',+ 'opus',+ 'ape',+ 'amr',+];+// Without FileViewerexport const ExceptiveType = ['xlsx',