Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(web): make audio attachments and references playable (#19842)

euvre947975d23137
diff --git a/web/src/components/document-preview/audio-preview.test.tsx b/web/src/components/document-preview/audio-preview.test.tsx
new file mode 100644
index 0000000000..ce4befe712
--- /dev/null
+++ b/web/src/components/document-preview/audio-preview.test.tsx
@@ -0,0 +1,100 @@
+import request from '@/utils/request';
+import { render, screen, waitFor } from '@testing-library/react';
+import { AudioPreviewer } from './audio-preview';
+
+jest.mock('@/utils/request', () => ({
+ __esModule: true,
+ default: jest.fn(),
+}));
+
+jest.mock('@/components/ui/spin', () => ({
+ Spin: () => <div data-testid="spin" />,
+}));
+
+const MockRequest = jest.mocked(request);
+
+beforeAll(() => {
+ URL.createObjectURL = jest.fn(
+ (blob: Blob | MediaSource) => `blob:${(blob as Blob).type}`,
+ );
+ URL.revokeObjectURL = jest.fn();
+});
+
+afterAll(() => {
+ jest.restoreAllMocks();
+});
+
+beforeEach(() => {
+ jest.clearAllMocks();
+});
+
+describe('AudioPreviewer', () => {
+ it('fetches the audio through the authenticated url and renders a player', async () => {
+ MockRequest.mockResolvedValue({
+ data: new Blob([new Uint8Array([1, 2, 3])], { type: 'audio/mpeg' }),
+ });
+
+ render(<AudioPreviewer url="/api/v1/agents/attachments/doc1/preview" />);
+
+ await waitFor(() =>
+ expect(screen.getByTestId('document-audio-player')).toBeInTheDocument(),
+ );
+ expect(MockRequest).toHaveBeenCalledWith(
+ '/api/v1/agents/attachments/doc1/preview',
+ { method: 'GET', responseType: 'blob' },
+ );
+ expect(
+ screen.getByTestId('document-audio-player').getAttribute('src'),
+ ).toBe('blob:audio/mpeg');
+ });
+
+ it('renders an error state and no player when the response carries no blob', async () => {
+ MockRequest.mockResolvedValue({ data: { code: 1999 } });
+
+ render(<AudioPreviewer url="/api/v1/agents/attachments/doc2/preview" />);
+
+ await waitFor(() =>
+ expect(screen.getByTestId('document-audio-error')).toBeInTheDocument(),
+ );
+ expect(
+ screen.queryByTestId('document-audio-player'),
+ ).not.toBeInTheDocument();
+ expect(screen.queryByTestId('spin')).not.toBeInTheDocument();
+ expect(URL.createObjectURL).not.toHaveBeenCalled();
+ });
+
+ it('ignores a stale response when the url changes mid-flight', async () => {
+ const pending: Record<string, (value: { data: Blob }) => void> = {};
+ MockRequest.mockImplementation(
+ (url: unknown) =>
+ new Promise((resolve) => {
+ pending[url as string] = resolve;
+ }),
+ );
+
+ const oldUrl = '/api/v1/agents/attachments/old/preview';
+ const newUrl = '/api/v1/agents/attachments/new/preview';
+ const { rerender } = render(<AudioPreviewer url={oldUrl} />);
+ rerender(<AudioPreviewer url={newUrl} />);
+
+ pending[oldUrl]({
+ data: new Blob([new Uint8Array([1])], { type: 'audio/old' }),
+ });
+
+ expect(
+ screen.queryByTestId('document-audio-player'),
+ ).not.toBeInTheDocument();
+ expect(URL.createObjectURL).not.toHaveBeenCalled();
+
+ pending[newUrl]({
+ data: new Blob([new Uint8Array([2])], { type: 'audio/new' }),
+ });
+
+ await waitFor(() =>
+ expect(screen.getByTestId('document-audio-player')).toBeInTheDocument(),
+ );
+ expect(
+ screen.getByTestId('document-audio-player').getAttribute('src'),
+ ).toBe('blob:audio/new');
+ });
+});
diff --git a/web/src/components/document-preview/audio-preview.tsx b/web/src/components/document-preview/audio-preview.tsx
new file mode 100644
index 0000000000..42e6c094c0
--- /dev/null
+++ b/web/src/components/document-preview/audio-preview.tsx
@@ -0,0 +1,101 @@
+import { Spin } from '@/components/ui/spin';
+import request from '@/utils/request';
+import classNames from 'classnames';
+import { useEffect, useState } from 'react';
+
+interface AudioPreviewerProps {
+ className?: string;
+ url: string;
+}
+
+export const AudioPreviewer: React.FC<AudioPreviewerProps> = ({
+ className,
+ url,
+}) => {
+ const [audioSrc, setAudioSrc] = useState<string | null>(null);
+ const [isLoading, setIsLoading] = useState<boolean>(true);
+ const [loadFailed, setLoadFailed] = useState<boolean>(false);
+
+ useEffect(() => {
+ if (!url) {
+ return;
+ }
+ let stale = false;
+ setIsLoading(true);
+ setLoadFailed(false);
+
+ const loadAudio = async () => {
+ try {
+ const res = await request(url, {
+ method: 'GET',
+ responseType: 'blob',
+ });
+ if (stale) {
+ return;
+ }
+ if (!(res.data instanceof Blob)) {
+ setLoadFailed(true);
+ return;
+ }
+ setAudioSrc(URL.createObjectURL(res.data));
+ } catch {
+ if (!stale) {
+ setLoadFailed(true);
+ }
+ } finally {
+ if (!stale) {
+ setIsLoading(false);
+ }
+ }
+ };
+
+ loadAudio();
+
+ return () => {
+ stale = true;
+ };
+ }, [url]);
+
+ useEffect(() => {
+ return () => {
+ if (audioSrc) {
+ URL.revokeObjectURL(audioSrc);
+ }
+ };
+ }, [audioSrc]);
+
+ return (
+ <div
+ className={classNames(
+ 'relative w-full h-full p-4 bg-background-paper border border-border-normal rounded-md audio-previewer',
+ className,
+ )}
+ >
+ {isLoading && (
+ <div className="absolute inset-0 flex items-center justify-center">
+ <Spin />
+ </div>
+ )}
+
+ {!isLoading && audioSrc && (
+ <div className="flex h-full items-center justify-center">
+ <audio
+ src={audioSrc}
+ controls
+ className="w-full max-w-2xl"
+ data-testid="document-audio-player"
+ />
+ </div>
+ )}
+
+ {!isLoading && !audioSrc && loadFailed && (
+ <div
+ className="flex h-full items-center justify-center text-text-secondary"
+ data-testid="document-audio-error"
+ >
+ Failed to load audio
+ </div>
+ )}
+ </div>
+ );
+};
diff --git a/web/src/components/document-preview/index.tsx b/web/src/components/document-preview/index.tsx
index 4e1637390c..b990e2f523 100644
--- a/web/src/components/document-preview/index.tsx
+++ b/web/src/components/document-preview/index.tsx
@@ -16,7 +16,8 @@
import { memo } from 'react';
-import { Images } from '@/constants/common';
+import { AudioExtensions, Images } from '@/constants/common';
+import { AudioPreviewer } from './audio-preview';
import CSVFileViewer from './csv-preview';
import { DocPreviewer } from './doc-preview';
import { EpubPreviewer } from './epub-preview';
@@ -88,6 +89,11 @@ const DocumentPreview = function ({
<VideoPreviewer className={className} url={url} />
</section>
)}
+ {AudioExtensions.indexOf(fileType) > -1 && (
+ <section>
+ <AudioPreviewer className={className} url={url} />
+ </section>
+ )}
{['ppt', 'pptx'].indexOf(fileType) > -1 && (
<section>
<PptPreviewer className={className} url={url} />
diff --git a/web/src/components/file-icon/index.tsx b/web/src/components/file-icon/index.tsx
index b113df10da..ece51eb3cb 100644
--- a/web/src/components/file-icon/index.tsx
+++ b/web/src/components/file-icon/index.tsx
@@ -34,7 +34,7 @@ const FileIcon = ({ name, id }: IProps) => {
const { data: fileThumbnails, setDocumentIds } =
useFetchDocumentThumbnailsByIds();
const fileThumbnail = fileThumbnails[id];
- const blobUrl = useAuthenticatedImageUrl(fileThumbnail);
+ const { src: blobUrl } = useAuthenticatedImageUrl(fileThumbnail);
useEffect(() => {
if (id) {
diff --git a/web/src/components/image/index.tsx b/web/src/components/image/index.tsx
index 0a01c7a567..e87ec9f41b 100644
--- a/web/src/components/image/index.tsx
+++ b/web/src/components/image/index.tsx
@@ -167,33 +167,42 @@ export const useDocumentImageUrl = (
return imageUrl;
};
+export type AuthenticatedImageUrlStatus = 'loading' | 'ready' | 'error';
+
/**
* Hook to convert any authenticated URL to a blob URL for use in <img> tags.
* Use this for thumbnail URLs or any other API URLs that require authentication.
+ * The status distinguishes an in-flight fetch from a failed one, which the
+ * empty src alone cannot express.
*/
-export const useAuthenticatedImageUrl = (url: string | undefined | null) => {
- const [imageUrl, setImageUrl] = useState<string>('');
+export const useAuthenticatedImageUrl = (
+ url: string | undefined | null,
+): { src: string; status: AuthenticatedImageUrlStatus } => {
+ const [state, setState] = useState<{
+ src: string;
+ status: AuthenticatedImageUrlStatus;
+ }>({ src: '', status: 'loading' });
useEffect(() => {
if (!url || !isAuthRequiredUrl(url)) {
- setImageUrl(url || '');
+ setState({ src: url || '', status: 'ready' });
return;
}
const authorization = getAuthorization();
let cancelled = false;
- setImageUrl('');
+ setState({ src: '', status: 'loading' });
const { promise, release } = fetchDocumentImage(url, authorization);
promise
.then((blobUrl) => {
if (!cancelled) {
- setImageUrl(blobUrl);
+ setState({ src: blobUrl, status: 'ready' });
}
})
.catch(() => {
if (!cancelled) {
- setImageUrl('');
+ setState({ src: '', status: 'error' });
}
});
@@ -203,7 +212,7 @@ export const useAuthenticatedImageUrl = (url: string | undefined | null) => {
};
}, [url]);
- return imageUrl;
+ return state;
};
/**
@@ -219,7 +228,7 @@ export const AuthenticatedImg = ({
}: React.ImgHTMLAttributes<HTMLImageElement> & {
fallback?: React.ReactNode;
}) => {
- const authenticatedSrc = useAuthenticatedImageUrl(src);
+ const { src: authenticatedSrc } = useAuthenticatedImageUrl(src);
if (!authenticatedSrc) return fallback ?? null;
diff --git a/web/src/components/next-message-item/uploaded-message-files.test.tsx b/web/src/components/next-message-item/uploaded-message-files.test.tsx
new file mode 100644
index 0000000000..ceb15272f2
--- /dev/null
+++ b/web/src/components/next-message-item/uploaded-message-files.test.tsx
@@ -0,0 +1,177 @@
+import { fireEvent, render, screen } from '@testing-library/react';
+import { UploadedMessageFiles } from './uploaded-message-files';
+import { isAudioFile } from './utils';
+
+jest.mock('../file-icon', () => ({
+ __esModule: true,
+ default: ({ name }: { name: string }) => (
+ <div data-testid="file-icon">{name}</div>
+ ),
+}));
+
+const mockUseAuthenticatedImageUrl = jest.fn(
+ (
+ url: string | null | undefined,
+ ): {
+ src: string;
+ status: 'loading' | 'ready' | 'error';
+ } => ({
+ src: url ?? '',
+ status: 'ready',
+ }),
+);
+
+jest.mock('../image', () => ({
+ __esModule: true,
+ useAuthenticatedImageUrl: (url: string | null | undefined) =>
+ mockUseAuthenticatedImageUrl(url),
+}));
+
+jest.mock('../svg-icon', () => ({
+ __esModule: true,
+ default: ({ name }: { name: string }) => (
+ <div data-testid="svg-icon">{name}</div>
+ ),
+}));
+
+jest.mock('../ui/modal/modal', () => ({
+ Modal: ({ open, title, children }: any) =>
+ open ? (
+ <div data-testid="audio-modal">
+ <div data-testid="audio-modal-title">{title}</div>
+ {children}
+ </div>
+ ) : null,
+}));
+
+jest.mock('../ui/spin', () => ({
+ Spin: () => <div data-testid="spin" />,
+}));
+
+jest.mock('react-photo-view', () => ({
+ PhotoProvider: ({ children }: any) => <>{children}</>,
+ PhotoView: ({ children }: any) => <>{children}</>,
+}));
+
+const uploadedAudio = {
+ created_at: 1789717195,
+ created_by: 'tenant-1',
+ extension: 'mp3',
+ id: 'attach-1',
+ mime_type: 'audio/mpeg',
+ name: '1.01a demo reading.mp3',
+ preview_url: null,
+ size: 1024,
+};
+
+beforeAll(() => {
+ URL.createObjectURL = jest.fn(() => 'blob:local-audio');
+ URL.revokeObjectURL = jest.fn();
+});
+
+describe('UploadedMessageFiles audio playback', () => {
+ it('detects audio files by mime type and by extension', () => {
+ expect(isAudioFile(uploadedAudio)).toBe(true);
+ expect(
+ isAudioFile({
+ ...uploadedAudio,
+ mime_type: '',
+ name: 'lesson.wav',
+ }),
+ ).toBe(true);
+ expect(
+ isAudioFile({
+ ...uploadedAudio,
+ mime_type: 'application/pdf',
+ name: 'lesson.pdf',
+ }),
+ ).toBe(false);
+ });
+
+ it('opens an inline player when an uploaded audio chip is clicked', () => {
+ render(
+ <UploadedMessageFiles files={[uploadedAudio]}></UploadedMessageFiles>,
+ );
+
+ expect(screen.queryByTestId('audio-modal')).not.toBeInTheDocument();
+
+ fireEvent.click(
+ screen.getByRole('button', { name: '1.01a demo reading.mp3' }),
+ );
+
+ const player = screen.getByTestId('uploaded-audio-player');
+ expect(player).toBeInTheDocument();
+ expect(player.getAttribute('controls')).not.toBeNull();
+ expect(player.getAttribute('src')).toContain(
+ '/api/v1/agents/attachments/attach-1/preview',
+ );
+ });
+
+ it('opens an inline player for an unsent local audio file', () => {
+ const localFile = new File([new Uint8Array([1])], 'note.mp3', {
+ type: 'audio/mpeg',
+ });
+
+ render(<UploadedMessageFiles files={[localFile]}></UploadedMessageFiles>);
+
+ fireEvent.click(screen.getByRole('button', { name: 'note.mp3' }));
+
+ const player = screen.getByTestId('uploaded-audio-player');
+ expect(player.getAttribute('src')).toBe('blob:local-audio');
+ });
+
+ it('does not open a player for image attachments', () => {
+ const image = {
+ ...uploadedAudio,
+ mime_type: 'image/png',
+ name: 'photo.png',
+ };
+
+ render(<UploadedMessageFiles files={[image]}></UploadedMessageFiles>);
+
+ fireEvent.click(screen.getByText('photo.png'));
+
+ expect(screen.queryByTestId('audio-modal')).not.toBeInTheDocument();
+ });
+
+ it('keeps the spinner while the remote audio preview is loading', () => {
+ mockUseAuthenticatedImageUrl.mockReturnValueOnce({
+ src: '',
+ status: 'loading',
+ });
+
+ render(
+ <UploadedMessageFiles files={[uploadedAudio]}></UploadedMessageFiles>,
+ );
+
+ fireEvent.click(
+ screen.getByRole('button', { name: '1.01a demo reading.mp3' }),
+ );
+
+ expect(screen.getByTestId('spin')).toBeInTheDocument();
+ expect(
+ screen.queryByTestId('uploaded-audio-player'),
+ ).not.toBeInTheDocument();
+ });
+
+ it('shows an error state instead of a spinner when the remote audio fetch fails', () => {
+ mockUseAuthenticatedImageUrl.mockReturnValueOnce({
+ src: '',
+ status: 'error',
+ });
+
+ render(
+ <UploadedMessageFiles files={[uploadedAudio]}></UploadedMessageFiles>,
+ );
+
+ fireEvent.click(
+ screen.getByRole('button', { name: '1.01a demo reading.mp3' }),
+ );
+
+ expect(screen.getByTestId('uploaded-audio-error')).toBeInTheDocument();
+ expect(
+ screen.queryByTestId('uploaded-audio-player'),
+ ).not.toBeInTheDocument();
+ expect(screen.queryByTestId('spin')).not.toBeInTheDocument();
+ });
+});
diff --git a/web/src/components/next-message-item/uploaded-message-files.tsx b/web/src/components/next-message-item/uploaded-message-files.tsx
index 0dea8e5799..24dae8d313 100644
--- a/web/src/components/next-message-item/uploaded-message-files.tsx
+++ b/web/src/components/next-message-item/uploaded-message-files.tsx
@@ -19,12 +19,15 @@ import { IDocumentInfo } from '@/interfaces/database/document';
import api from '@/utils/api';
import { getExtension } from '@/utils/document-util';
import { formatBytes } from '@/utils/file-util';
+import { cn } from '@/lib/utils';
import { memo, useEffect, useState } from 'react';
import { PhotoProvider, PhotoView } from 'react-photo-view';
import FileIcon from '../file-icon';
import { useAuthenticatedImageUrl } from '../image';
+import { Modal } from '../ui/modal/modal';
+import { Spin } from '../ui/spin';
import SvgIcon from '../svg-icon';
-import { getFileMimeType, isImageFile } from './utils';
+import { getFileMimeType, isAudioFile, isImageFile } from './utils';
interface IProps {
files?: File[] | IDocumentInfo[] | UploadResponseDataType[];
@@ -43,7 +46,7 @@ function UploadedFileImage({
name: string;
mimeType?: string;
}) {
- const src = useAuthenticatedImageUrl(
+ const { src } = useAuthenticatedImageUrl(
id
? api.getAttachmentFilePreview({ docId: id, filename: name, mimeType })
: null,
@@ -89,6 +92,79 @@ function LocalFileImage({ file, name }: { file: File; name: string }) {
);
}
+// Audio attachments have no visual content, so the chip opens an inline
+// player instead of a lightbox: chat uploads live in the per-user downloads
+// bucket and are fetched through the authenticated attachment preview
+// endpoint, while unsent local files play straight from an object URL.
+function AudioPlayerModal({
+ file,
+ onClose,
+}: {
+ file: File | UploadResponseDataType | IDocumentInfo;
+ onClose: () => void;
+}) {
+ const name = file.name;
+ const isLocal = file instanceof File;
+ const { src: remoteSrc, status: remoteStatus } = useAuthenticatedImageUrl(
+ !isLocal && file.id
+ ? api.getAttachmentFilePreview({
+ docId: file.id,
+ filename: name,
+ mimeType: getFileMimeType(file),
+ })
+ : null,
+ );
+ const [localSrc, setLocalSrc] = useState('');
+
+ useEffect(() => {
+ if (isLocal) {
+ const url = URL.createObjectURL(file);
+ setLocalSrc(url);
+ return () => URL.revokeObjectURL(url);
+ }
+ }, [isLocal, file]);
+
+ const src = isLocal ? localSrc : remoteSrc;
+
+ return (
+ <Modal
+ open
+ onCancel={onClose}
+ showfooter={false}
+ title={
+ <div className="flex items-center gap-2">
+ {isLocal ? (
+ <SvgIcon name={`file-icon/${getExtension(name)}`} width={24} />
+ ) : (
+ <FileIcon id={file.id ?? ''} name={name}></FileIcon>
+ )}
+ <span className="truncate">{name}</span>
+ </div>
+ }
+ >
+ {src ? (
+ <audio
+ controls
+ src={src}
+ className="w-full"
+ data-testid="uploaded-audio-player"
+ />
+ ) : remoteStatus === 'error' ? (
+ <div
+ className="flex items-center justify-center py-6 text-text-secondary"
+ data-testid="uploaded-audio-error"
+ >
+ Failed to load audio
+ </div>
+ ) : (
+ <div className="flex items-center justify-center py-6">
+ <Spin />
+ </div>
+ )}
+ </Modal>
+ );
+}
+
type NameWidgetType = {
name: string;
size: number;
@@ -109,6 +185,10 @@ function NameWidget({ name, size }: NameWidgetType) {
);
}
export function InnerUploadedMessageFiles({ files = [] }: IProps) {
+ const [playingFile, setPlayingFile] = useState<
+ File | UploadResponseDataType | IDocumentInfo | null
+ >(null);
+
return (
<PhotoProvider>
<section className="flex gap-2 pt-2 flex-wrap">
@@ -116,9 +196,30 @@ export function InnerUploadedMessageFiles({ files = [] }: IProps) {
const name = file.name;
const isFile = file instanceof File;
const isImage = isImageFile(file);
+ const isAudio = isAudioFile(file);
return (
- <div key={idx} className="flex gap-1 border rounded-md p-1.5">
+ <div
+ key={idx}
+ className={cn(
+ 'flex gap-1 border rounded-md p-1.5',
+ isAudio && 'cursor-pointer hover:border-accent-primary',
+ )}
+ role={isAudio ? 'button' : undefined}
+ tabIndex={isAudio ? 0 : undefined}
+ aria-label={isAudio ? name : undefined}
+ onClick={isAudio ? () => setPlayingFile(file) : undefined}
+ onKeyDown={
+ isAudio
+ ? (e) => {
+ if (e.key === 'Enter' || e.key === ' ') {
+ e.preventDefault();
+ setPlayingFile(file);
+ }
+ }
+ : undefined
+ }
+ >
{isImage ? (
isFile ? (
<LocalFileImage file={file} name={name}></LocalFileImage>
@@ -146,6 +247,12 @@ export function InnerUploadedMessageFiles({ files = [] }: IProps) {
);
})}
</section>
+ {playingFile && (
+ <AudioPlayerModal
+ file={playingFile}
+ onClose={() => setPlayingFile(null)}
+ ></AudioPlayerModal>
+ )}
</PhotoProvider>
);
}
diff --git a/web/src/components/next-message-item/utils.ts b/web/src/components/next-message-item/utils.ts
index f8d637b01d..28273eb802 100644
--- a/web/src/components/next-message-item/utils.ts
+++ b/web/src/components/next-message-item/utils.ts
@@ -14,6 +14,7 @@
* limitations under the License.
*/
+import { AudioExtensions } from '@/constants/common';
import { UploadResponseDataType } from '@/interfaces/database/chat';
import { IDocumentInfo } from '@/interfaces/database/document';
import { currentReg, parseCitationIndex } from '@/utils/chat';
@@ -67,3 +68,13 @@ export function isImageFile(
}
return ImageExtensions.includes(getExtension(file.name));
}
+
+export function isAudioFile(
+ file: File | IDocumentInfo | UploadResponseDataType,
+): boolean {
+ const mimeType = getFileMimeType(file);
+ if (mimeType) {
+ return mimeType.startsWith('audio/');
+ }
+ return AudioExtensions.includes(getExtension(file.name));
+}
diff --git a/web/src/constants/common.ts b/web/src/constants/common.ts
index 88bbb231af..62bc349ec3 100644
--- a/web/src/constants/common.ts
+++ b/web/src/constants/common.ts
@@ -164,6 +164,25 @@ export const Images = [
'ico',
];
+export const AudioExtensions = [
+ 'mp3',
+ 'wav',
+ 'wave',
+ 'aac',
+ 'flac',
+ 'ogg',
+ 'oga',
+ 'm4a',
+ 'wma',
+ 'aiff',
+ 'aif',
+ 'au',
+ 'midi',
+ 'opus',
+ 'ape',
+ 'amr',
+];
+
// Without FileViewer
export const ExceptiveType = [
'xlsx',