RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
import { AudioRecorder, useAudioRecorder } from 'react-audio-voice-recorder';
import { Button } from '@/components/ui/button';
import { Authorization } from '@/constants/authorization';
import { cn } from '@/lib/utils';
import api from '@/utils/api';
import { getAuthorization } from '@/utils/authorization-util';
import { chain, sum } from 'lodash';
import { Loader2, Mic } from 'lucide-react';
import { useCallback, useEffect, useRef, useState } from 'react';
import message from './message';
const VoiceVisualizer = ({ isRecording }: { isRecording: boolean }) => {
const canvasRef = useRef<HTMLCanvasElement>(null);
const audioContextRef = useRef<AudioContext | null>(null);
const analyserRef = useRef<AnalyserNode | null>(null);
const animationFrameRef = useRef<number>(0);
const streamRef = useRef<MediaStream | null>(null);
const draw = useCallback(() => {
const canvas = canvasRef.current;
if (!canvas) return;
const ctx = canvas.getContext('2d');
if (!ctx) return;
const analyser = analyserRef.current;
if (!analyser) return;
// Set canvas dimensions
const width = canvas.clientWidth * window.devicePixelRatio;
const height = canvas.clientHeight * window.devicePixelRatio;
const centerY = height / 2;
if (canvas.width !== width || canvas.height !== height) {
canvas.width = width;
canvas.height = height;
}
// Clear canvas
ctx.clearRect(0, 0, width, height);
// Get frequency data
const bufferLength = analyser.frequencyBinCount;
const dataArray = new Uint8Array(bufferLength);
/**
* The frequencies are spread linearly from 0Hz to 1/2 of the sample rate
* @see https://developer.mozilla.org/en-US/docs/Web/API/AnalyserNode/getByteFrequencyData
*/
analyser.getByteFrequencyData(dataArray);
const desiredBarCount = 6;
const freqSpanPerPole =
analyser.context.sampleRate / 2 / analyser.frequencyBinCount;
const freqMinIndex = Math.max(1, Math.floor(100 / freqSpanPerPole));
const freqMaxIndex =
Math.max(freqMinIndex, Math.ceil(3500 / freqSpanPerPole)) + 1;
const freqData = dataArray.slice(
freqMinIndex,
Math.max(freqMinIndex + desiredBarCount, freqMaxIndex),
);
const reducedFreqData = chain(freqData)
.chunk(Math.floor(freqData.length / desiredBarCount))
.take(desiredBarCount)
.map(sum)
.value();
// Draw waveform
const barGap = 1 * window.devicePixelRatio;
const barWidth = (width - barGap * (desiredBarCount - 1)) / desiredBarCount;
// Create gradient
const gradient = ctx.createLinearGradient(0, 0, 0, height);
gradient.addColorStop(0, '#3ba05c'); // Blue
gradient.addColorStop(1, '#3ba05c'); // Light blue
// gradient.addColorStop(0, isDark ? '#fff' : '#000'); // Blue
// gradient.addColorStop(1, isDark ? '#eee' : '#eee'); // Light blue
for (let i = 0, x = 0; i < desiredBarCount; i++, x += barWidth + barGap) {
const barHeight = (reducedFreqData[i] / 255) * centerY;
ctx.fillStyle = gradient;
ctx.fillRect(x, centerY - barHeight, barWidth, barHeight * 2);
}
animationFrameRef.current = requestAnimationFrame(draw);
}, []);
const startVisualization = useCallback(async () => {
try {
// Check if the browser supports getUserMedia
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
console.error('Browser does not support getUserMedia API');
return;
}
// Request microphone permission
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
streamRef.current = stream;
// Create audio context and analyzer
const audioContext = new (
window.AudioContext || (window as any).webkitAudioContext
)();
audioContextRef.current = audioContext;
const analyser = audioContext.createAnalyser();
analyserRef.current = analyser;
analyser.fftSize = 256;
// Connect audio nodes
const source = audioContext.createMediaStreamSource(stream);
source.connect(analyser);
// Start drawing
draw();
} catch (error) {
console.error(
'Unable to access microphone for voice visualization:',
error,
);
}
}, [draw]);
const stopVisualization = useCallback(() => {
// Stop animation frame
if (animationFrameRef.current) {
cancelAnimationFrame(animationFrameRef.current);
}
// Stop audio stream
if (streamRef.current) {
streamRef.current.getTracks().forEach((track) => track.stop());
}
// Close audio context
if (audioContextRef.current && audioContextRef.current.state !== 'closed') {
audioContextRef.current.close();
}
// Clear canvas
const canvas = canvasRef.current;
if (canvas) {
const ctx = canvas.getContext('2d');
if (ctx) {
ctx.clearRect(0, 0, canvas.width, canvas.height);
}
}
}, []);
useEffect(() => {
if (isRecording) {
startVisualization();
} else {
stopVisualization();
}
return () => {
stopVisualization();
};
}, [isRecording, startVisualization, stopVisualization]);
return <canvas ref={canvasRef} className="block size-full" />;
};
export const AudioButton = ({
onOk,
testId,
}: {
onOk?: (transcript: string) => void;
testId?: string;
}) => {
// const [showInputBox, setShowInputBox] = useState(false);
const [isRecording, setIsRecording] = useState(false);
const [isProcessing, setIsProcessing] = useState(false);
const [, setRecordingTime] = useState(0);
const [, setTranscript] = useState('');
const [, setPopoverOpen] = useState(false);
const recorderControls = useAudioRecorder();
const intervalRef = useRef<NodeJS.Timeout | null>(null);
// Handle logic after recording is complete
const handleRecordingComplete = async (blob: Blob) => {
setIsRecording(false);
// const url = URL.createObjectURL(blob);
// const a = document.createElement('a');
// a.href = url;
// a.download = 'recording.webm';
// document.body.appendChild(a);
// a.click();
setIsProcessing(true);
if (intervalRef.current) {
clearInterval(intervalRef.current);
intervalRef.current = null;
}
try {
const audioFile = new File([blob], 'recording.webm', {
type: blob.type || 'audio/webm',
// type: 'audio/mpeg',
});
const formData = new FormData();
formData.append('file', audioFile);
formData.append('stream', 'false');
const response = await fetch(api.chatsTranscriptions, {
method: 'POST',
headers: {
[Authorization]: getAuthorization(),
// 'Content-Type': blob.type || 'audio/webm',
},
body: formData,
});
// if (!response.ok) {
// throw new Error(`HTTP error! status: ${response.status}`);
// }
// if (!response.body) {
// throw new Error('ReadableStream not supported in this browser');
// }
const { data, code } = await response.json();
if (code === 0 && data && data.text) {
setTranscript(data.text);
onOk?.(data.text);
}
setPopoverOpen(false);
} catch (error) {
console.error('Failed to process audio:', error);
// setTranscript(t('voiceRecorder.processingError'));
} finally {
setIsProcessing(false);
}
};
// Start recording
const startRecording = () => {
if (
!navigator.mediaDevices ||
typeof navigator.mediaDevices.getUserMedia !== 'function'
) {
message.error('Browser does not support getUserMedia API');
return;
}
recorderControls.startRecording();
setIsRecording(true);
// setShowInputBox(true);
setPopoverOpen(true);
setRecordingTime(0);
// Start timing
if (intervalRef.current) {
clearInterval(intervalRef.current);
}
intervalRef.current = setInterval(() => {
setRecordingTime((prev) => prev + 1);
}, 1000);
};
// Stop recording
const stopRecording = () => {
recorderControls.stopRecording();
setIsRecording(false);
// setShowInputBox(false);
setPopoverOpen(false);
setRecordingTime(0);
// Clear timer
if (intervalRef.current) {
clearInterval(intervalRef.current);
intervalRef.current = null;
}
};
// Clear transcription content
// const clearTranscript = () => {
// setTranscript('');
// };
useEffect(() => {
return () => {
if (intervalRef.current) {
clearInterval(intervalRef.current);
}
};
}, []);
return (
<div>
<div className=" relative w-6 h-6 flex items-center justify-center">
{isRecording && (
<div className="absolute inset-0 size-full overflow-hidden flex items-center justify-center p-1">
<VoiceVisualizer isRecording={isRecording} />
</div>
)}
{isRecording && (
<div
className="
absolute inset-0 rounded-full border-2 border-state-success
animate-ping opacity-75 pointer-events-none"
/>
)}
<Button
type="button"
variant="transparent"
size="icon-xs"
// onMouseDown={() => {
// startRecording();
// }}
// onMouseUp={() => {
// stopRecording();
// }}
onClick={() => {
if (isRecording) {
stopRecording();
} else {
startRecording();
}
}}
className={cn(
'border-0 p-2 rounded-md border-none bg-transparent hover:bg-state-success-5',
isRecording &&
'animate-pulse !bg-state-success/20 text-state-success rounded-full',
)}
disabled={isProcessing}
data-testid={testId}
>
{isProcessing ? (
<Loader2 size={16} className=" animate-spin" />
) : isRecording ? (
<></>
) : (
// <Mic size={16} className="text-text-primary" />
// <Square size={12} className="text-text-primary" />
<Mic size={16} />
)}
</Button>
</div>
{/* Hide original component */}
<div className="hidden">
<AudioRecorder
onRecordingComplete={handleRecordingComplete}
recorderControls={recorderControls}
/>
</div>
</div>
);
};