Skip to content

The open-source AI voice studio. Clone, dictate, create.

macos audio capture for sample creation

Jamie Pine446182e16c8d
diff --git a/app/src/App.tsx b/app/src/App.tsx
index 85d51597..623f458a 100644
--- a/app/src/App.tsx
+++ b/app/src/App.tsx
@@ -14,7 +14,7 @@ import { UpdateNotification } from '@/components/UpdateNotification';
import { Toaster } from '@/components/ui/toaster';
import { ProfileList } from '@/components/VoiceProfiles/ProfileList';
import { useModelDownloadToast } from '@/lib/hooks/useModelDownloadToast';
-import { useRestoreActiveTasks, MODEL_DISPLAY_NAMES } from '@/lib/hooks/useRestoreActiveTasks';
+import { MODEL_DISPLAY_NAMES, useRestoreActiveTasks } from '@/lib/hooks/useRestoreActiveTasks';
import { isMacOS, isTauri, setupWindowCloseHandler, startServer } from '@/lib/tauri';
// Track if server is starting to prevent duplicate starts
diff --git a/app/src/components/VoiceProfiles/ProfileForm.tsx b/app/src/components/VoiceProfiles/ProfileForm.tsx
index d01681d6..f67c163f 100644
--- a/app/src/components/VoiceProfiles/ProfileForm.tsx
+++ b/app/src/components/VoiceProfiles/ProfileForm.tsx
@@ -39,9 +39,11 @@ import {
} from '@/lib/hooks/useProfiles';
import { useTranscription } from '@/lib/hooks/useTranscription';
import { useAudioRecording } from '@/lib/hooks/useAudioRecording';
+import { useSystemAudioCapture } from '@/lib/hooks/useSystemAudioCapture';
import { useUIStore } from '@/stores/uiStore';
-import { Mic, Square, Upload } from 'lucide-react';
+import { Mic, Square, Upload, Monitor } from 'lucide-react';
import { formatAudioDuration } from '@/lib/utils/audio';
+import { isTauri } from '@/lib/tauri';
// Helper function to get audio duration from File
async function getAudioDuration(file: File): Promise<number> {
@@ -101,7 +103,7 @@ export function ProfileForm() {
const addSample = useAddSample();
const transcribe = useTranscription();
const { toast } = useToast();
- const [sampleMode, setSampleMode] = useState<'upload' | 'record'>('upload');
+ const [sampleMode, setSampleMode] = useState<'upload' | 'record' | 'system'>('upload');
const [audioDuration, setAudioDuration] = useState<number | null>(null);
const [isValidatingAudio, setIsValidatingAudio] = useState(false);
const isCreating = !editingProfileId;
@@ -171,6 +173,28 @@ export function ProfileForm() {
},
});
+ const {
+ isRecording: isSystemRecording,
+ duration: systemDuration,
+ error: systemRecordingError,
+ isSupported: isSystemAudioSupported,
+ startRecording: startSystemRecording,
+ stopRecording: stopSystemRecording,
+ cancelRecording: cancelSystemRecording,
+ } = useSystemAudioCapture({
+ maxDurationSeconds: 30,
+ onRecordingComplete: (blob) => {
+ const file = new File([blob], `system-audio-${Date.now()}.wav`, {
+ type: blob.type || 'audio/wav',
+ });
+ form.setValue('sampleFile', file, { shouldValidate: true });
+ toast({
+ title: 'System audio captured',
+ description: 'Audio has been captured successfully.',
+ });
+ },
+ });
+
// Show recording errors
useEffect(() => {
if (recordingError) {
@@ -182,6 +206,17 @@ export function ProfileForm() {
}
}, [recordingError, toast]);
+ // Show system audio recording errors
+ useEffect(() => {
+ if (systemRecordingError) {
+ toast({
+ title: 'System audio capture error',
+ description: systemRecordingError,
+ variant: 'destructive',
+ });
+ }
+ }, [systemRecordingError, toast]);
+
useEffect(() => {
if (editingProfile) {
form.reset({
@@ -234,7 +269,11 @@ export function ProfileForm() {
}
function handleCancelRecording() {
- cancelRecording();
+ if (sampleMode === 'record') {
+ cancelRecording();
+ } else if (sampleMode === 'system') {
+ cancelSystemRecording();
+ }
form.resetField('sampleFile');
}
@@ -345,6 +384,9 @@ export function ProfileForm() {
if (isRecording) {
cancelRecording();
}
+ if (isSystemRecording) {
+ cancelSystemRecording();
+ }
}
}
@@ -432,9 +474,19 @@ export function ProfileForm() {
<Tabs
value={sampleMode}
- onValueChange={(v) => setSampleMode(v as 'upload' | 'record')}
+ onValueChange={(v) => {
+ const newMode = v as 'upload' | 'record' | 'system';
+ // Cancel any active recordings when switching modes
+ if (isRecording && newMode !== 'record') {
+ cancelRecording();
+ }
+ if (isSystemRecording && newMode !== 'system') {
+ cancelSystemRecording();
+ }
+ setSampleMode(newMode);
+ }}
>
- <TabsList className="grid w-full grid-cols-2">
+ <TabsList className={`grid w-full ${isTauri() && isSystemAudioSupported ? 'grid-cols-3' : 'grid-cols-2'}`}>
<TabsTrigger value="upload" className="flex items-center gap-2">
<Upload className="h-4 w-4" />
Upload
@@ -443,6 +495,12 @@ export function ProfileForm() {
<Mic className="h-4 w-4" />
Record
</TabsTrigger>
+ {isTauri() && isSystemAudioSupported && (
+ <TabsTrigger value="system" className="flex items-center gap-2">
+ <Monitor className="h-4 w-4" />
+ System Audio
+ </TabsTrigger>
+ )}
</TabsList>
<TabsContent value="upload" className="space-y-4">
@@ -611,6 +669,103 @@ export function ProfileForm() {
)}
/>
</TabsContent>
+
+ {isTauri() && isSystemAudioSupported && (
+ <TabsContent value="system" className="space-y-4">
+ <FormField
+ control={form.control}
+ name="sampleFile"
+ render={() => (
+ <FormItem>
+ <FormLabel>Capture System Audio</FormLabel>
+ <FormControl>
+ <div className="space-y-4">
+ {!isSystemRecording && !selectedFile && (
+ <div className="flex flex-col items-center gap-4 p-4 border-2 border-dashed rounded-lg">
+ <Button
+ type="button"
+ onClick={startSystemRecording}
+ size="lg"
+ className="flex items-center gap-2"
+ >
+ <Monitor className="h-5 w-5" />
+ Start Capture
+ </Button>
+ <p className="text-sm text-muted-foreground text-center">
+ Capture audio from your system. Maximum duration: 30 seconds.
+ </p>
+ </div>
+ )}
+
+ {isSystemRecording && (
+ <div className="flex flex-col items-center gap-4 p-4 border-2 border-destructive rounded-lg bg-destructive/5">
+ <div className="flex items-center gap-4">
+ <div className="flex items-center gap-2">
+ <div className="h-3 w-3 rounded-full bg-destructive animate-pulse" />
+ <span className="text-lg font-mono font-semibold">
+ {formatAudioDuration(systemDuration)}
+ </span>
+ </div>
+ </div>
+ <Button
+ type="button"
+ onClick={stopSystemRecording}
+ variant="destructive"
+ className="flex items-center gap-2"
+ >
+ <Square className="h-4 w-4" />
+ Stop Capture
+ </Button>
+ <p className="text-sm text-muted-foreground text-center">
+ Capturing system audio... ({formatAudioDuration(30 - systemDuration)}{' '}
+ remaining)
+ </p>
+ </div>
+ )}
+
+ {selectedFile && !isSystemRecording && (
+ <div className="flex flex-col items-center gap-4 p-4 border-2 border-primary rounded-lg bg-primary/5">
+ <div className="flex items-center gap-2">
+ <Monitor className="h-5 w-5 text-primary" />
+ <span className="font-medium">Capture complete</span>
+ </div>
+ <p className="text-sm text-muted-foreground text-center">
+ File: {selectedFile.name}
+ </p>
+ <div className="flex gap-2">
+ <Button
+ type="button"
+ variant="outline"
+ onClick={handleTranscribe}
+ disabled={transcribe.isPending}
+ className="flex items-center gap-2"
+ >
+ <Mic className="h-4 w-4" />
+ {transcribe.isPending ? 'Transcribing...' : 'Transcribe'}
+ </Button>
+ <Button
+ type="button"
+ variant="outline"
+ onClick={handleCancelRecording}
+ className="flex items-center gap-2"
+ >
+ Capture Again
+ </Button>
+ </div>
+ </div>
+ )}
+ </div>
+ </FormControl>
+ <FormDescription>
+ Capture audio from your system (speakers, applications). Maximum duration is 30
+ seconds.
+ </FormDescription>
+ <FormMessage />
+ </FormItem>
+ )}
+ />
+ </TabsContent>
+ )}
</Tabs>
<FormField
diff --git a/app/src/components/VoiceProfiles/SampleUpload.tsx b/app/src/components/VoiceProfiles/SampleUpload.tsx
index a443634f..3cc3afa6 100644
--- a/app/src/components/VoiceProfiles/SampleUpload.tsx
+++ b/app/src/components/VoiceProfiles/SampleUpload.tsx
@@ -26,8 +26,10 @@ import { useToast } from '@/components/ui/use-toast';
import { useAddSample, useProfile } from '@/lib/hooks/useProfiles';
import { useTranscription } from '@/lib/hooks/useTranscription';
import { useAudioRecording } from '@/lib/hooks/useAudioRecording';
-import { Mic, Square, Upload } from 'lucide-react';
+import { useSystemAudioCapture } from '@/lib/hooks/useSystemAudioCapture';
+import { Mic, Square, Upload, Monitor } from 'lucide-react';
import { formatAudioDuration } from '@/lib/utils/audio';
+import { isTauri } from '@/lib/tauri';
const sampleSchema = z.object({
file: z.instanceof(File, { message: 'Please select an audio file' }),
@@ -50,7 +52,7 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
const transcribe = useTranscription();
const { data: profile } = useProfile(profileId);
const { toast } = useToast();
- const [mode, setMode] = useState<'upload' | 'record'>('upload');
+ const [mode, setMode] = useState<'upload' | 'record' | 'system'>('upload');
const form = useForm<SampleFormValues>({
resolver: zodResolver(sampleSchema),
@@ -83,6 +85,29 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
},
});
+ const {
+ isRecording: isSystemRecording,
+ duration: systemDuration,
+ error: systemRecordingError,
+ isSupported: isSystemAudioSupported,
+ startRecording: startSystemRecording,
+ stopRecording: stopSystemRecording,
+ cancelRecording: cancelSystemRecording,
+ } = useSystemAudioCapture({
+ maxDurationSeconds: 30,
+ onRecordingComplete: (blob) => {
+ // Convert blob to File object
+ const file = new File([blob], `system-audio-${Date.now()}.wav`, {
+ type: blob.type || 'audio/wav',
+ });
+ form.setValue('file', file, { shouldValidate: true });
+ toast({
+ title: 'System audio captured',
+ description: 'Audio has been captured successfully.',
+ });
+ },
+ });
+
// Show recording errors
useEffect(() => {
if (recordingError) {
@@ -94,6 +119,17 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
}
}, [recordingError, toast]);
+ // Show system audio recording errors
+ useEffect(() => {
+ if (systemRecordingError) {
+ toast({
+ title: 'System audio capture error',
+ description: systemRecordingError,
+ variant: 'destructive',
+ });
+ }
+ }, [systemRecordingError, toast]);
+
async function handleTranscribe() {
const file = form.getValues('file');
if (!file) {
@@ -154,12 +190,19 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
if (isRecording) {
cancelRecording();
}
+ if (isSystemRecording) {
+ cancelSystemRecording();
+ }
}
onOpenChange(newOpen);
}
function handleCancelRecording() {
- cancelRecording();
+ if (mode === 'record') {
+ cancelRecording();
+ } else if (mode === 'system') {
+ cancelSystemRecording();
+ }
// Reset file field by clearing the input
form.resetField('file');
}
@@ -176,8 +219,13 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
<Form {...form}>
<form onSubmit={form.handleSubmit(onSubmit)} className="space-y-4">
- <Tabs value={mode} onValueChange={(v) => setMode(v as 'upload' | 'record')}>
- <TabsList className="grid w-full grid-cols-2">
+ <Tabs
+ value={mode}
+ onValueChange={(v) => setMode(v as 'upload' | 'record' | 'system')}
+ >
+ <TabsList
+ className={`grid w-full ${isTauri() && isSystemAudioSupported ? 'grid-cols-3' : 'grid-cols-2'}`}
+ >
<TabsTrigger value="upload" className="flex items-center gap-2">
<Upload className="h-4 w-4" />
Upload
@@ -186,6 +234,12 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
<Mic className="h-4 w-4" />
Record
</TabsTrigger>
+ {isTauri() && isSystemAudioSupported && (
+ <TabsTrigger value="system" className="flex items-center gap-2">
+ <Monitor className="h-4 w-4" />
+ System Audio
+ </TabsTrigger>
+ )}
</TabsList>
<TabsContent value="upload" className="space-y-4">
@@ -325,6 +379,103 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
)}
/>
</TabsContent>
+
+ {isTauri() && isSystemAudioSupported && (
+ <TabsContent value="system" className="space-y-4">
+ <FormField
+ control={form.control}
+ name="file"
+ render={() => (
+ <FormItem>
+ <FormLabel>Capture System Audio</FormLabel>
+ <FormControl>
+ <div className="space-y-4">
+ {!isSystemRecording && !selectedFile && (
+ <div className="flex flex-col items-center gap-4 p-6 border-2 border-dashed rounded-lg">
+ <Button
+ type="button"
+ onClick={startSystemRecording}
+ size="lg"
+ className="flex items-center gap-2"
+ >
+ <Monitor className="h-5 w-5" />
+ Start Capture
+ </Button>
+ <p className="text-sm text-muted-foreground text-center">
+ Capture audio from your system. Maximum duration: 30 seconds.
+ </p>
+ </div>
+ )}
+
+ {isSystemRecording && (
+ <div className="flex flex-col items-center gap-4 p-6 border-2 border-destructive rounded-lg bg-destructive/5">
+ <div className="flex items-center gap-4">
+ <div className="flex items-center gap-2">
+ <div className="h-3 w-3 rounded-full bg-destructive animate-pulse" />
+ <span className="text-lg font-mono font-semibold">
+ {formatAudioDuration(systemDuration)}
+ </span>
+ </div>
+ </div>
+ <Button
+ type="button"
+ onClick={stopSystemRecording}
+ variant="destructive"
+ className="flex items-center gap-2"
+ >
+ <Square className="h-4 w-4" />
+ Stop Capture
+ </Button>
+ <p className="text-sm text-muted-foreground text-center">
+ Capturing system audio... ({formatAudioDuration(30 - systemDuration)}{' '}
+ remaining)
+ </p>
+ </div>
+ )}
+
+ {selectedFile && !isSystemRecording && mode === 'system' && (
+ <div className="flex flex-col items-center gap-4 p-6 border-2 border-primary rounded-lg bg-primary/5">
+ <div className="flex items-center gap-2">
+ <Monitor className="h-5 w-5 text-primary" />
+ <span className="font-medium">Capture complete</span>
+ </div>
+ <p className="text-sm text-muted-foreground">
+ File: {selectedFile.name}
+ </p>
+ <div className="flex gap-2">
+ <Button
+ type="button"
+ variant="outline"
+ onClick={handleTranscribe}
+ disabled={transcribe.isPending}
+ className="flex items-center gap-2"
+ >
+ <Mic className="h-4 w-4" />
+ {transcribe.isPending ? 'Transcribing...' : 'Transcribe'}
+ </Button>
+ <Button
+ type="button"
+ variant="outline"
+ onClick={handleCancelRecording}
+ className="flex items-center gap-2"
+ >
+ Capture Again
+ </Button>
+ </div>
+ </div>
+ )}
+ </div>
+ </FormControl>
+ <FormDescription>
+ Capture audio from your system (applications, browser tabs, etc.). Works
+ natively without browser dialogs.
+ </FormDescription>
+ <FormMessage />
+ </FormItem>
+ )}
+ />
+ </TabsContent>
+ )}
</Tabs>
<FormField
diff --git a/app/src/lib/hooks/useSystemAudioCapture.ts b/app/src/lib/hooks/useSystemAudioCapture.ts
new file mode 100644
index 00000000..13a22b55
--- /dev/null
+++ b/app/src/lib/hooks/useSystemAudioCapture.ts
@@ -0,0 +1,165 @@
+import { useState, useRef, useCallback, useEffect } from 'react';
+import { invoke } from '@tauri-apps/api/core';
+import { isTauri } from '@/lib/tauri';
+
+interface UseSystemAudioCaptureOptions {
+ maxDurationSeconds?: number;
+ onRecordingComplete?: (blob: Blob) => void;
+}
+
+/**
+ * Hook for native system audio capture using Tauri commands.
+ * Uses ScreenCaptureKit on macOS and WASAPI loopback on Windows.
+ */
+export function useSystemAudioCapture({
+ maxDurationSeconds = 30,
+ onRecordingComplete,
+}: UseSystemAudioCaptureOptions = {}) {
+ const [isRecording, setIsRecording] = useState(false);
+ const [duration, setDuration] = useState(0);
+ const [error, setError] = useState<string | null>(null);
+ const [isSupported, setIsSupported] = useState(false);
+ const timerRef = useRef<number | null>(null);
+ const startTimeRef = useRef<number | null>(null);
+ const stopRecordingRef = useRef<(() => Promise<void>) | null>(null);
+
+ // Check if system audio capture is supported
+ useEffect(() => {
+ if (!isTauri()) {
+ setIsSupported(false);
+ return;
+ }
+
+ invoke<boolean>('is_system_audio_supported')
+ .then((supported) => {
+ setIsSupported(supported);
+ })
+ .catch(() => {
+ setIsSupported(false);
+ });
+ }, []);
+
+ const startRecording = useCallback(async () => {
+ if (!isTauri()) {
+ const errorMsg = 'System audio capture is only available in the desktop app.';
+ setError(errorMsg);
+ return;
+ }
+
+ if (!isSupported) {
+ const errorMsg = 'System audio capture is not supported on this platform.';
+ setError(errorMsg);
+ return;
+ }
+
+ try {
+ setError(null);
+ setDuration(0);
+
+ // Start native capture
+ await invoke('start_system_audio_capture', {
+ maxDurationSecs: maxDurationSeconds,
+ });
+
+ setIsRecording(true);
+ startTimeRef.current = Date.now();
+
+ // Start timer
+ timerRef.current = window.setInterval(() => {
+ if (startTimeRef.current) {
+ const elapsed = (Date.now() - startTimeRef.current) / 1000;
+ setDuration(elapsed);
+
+ // Auto-stop at max duration
+ if (elapsed >= maxDurationSeconds && stopRecordingRef.current) {
+ void stopRecordingRef.current();
+ }
+ }
+ }, 100);
+ } catch (err) {
+ const errorMessage =
+ err instanceof Error
+ ? err.message
+ : 'Failed to start system audio capture. Please check permissions.';
+ setError(errorMessage);
+ setIsRecording(false);
+ }
+ }, [maxDurationSeconds, isSupported]);
+
+ const stopRecording = useCallback(async () => {
+ if (!isRecording || !isTauri()) {
+ return;
+ }
+
+ try {
+ setIsRecording(false);
+
+ if (timerRef.current !== null) {
+ clearInterval(timerRef.current);
+ timerRef.current = null;
+ }
+
+ // Stop capture and get base64 WAV data
+ const base64Data = await invoke<string>('stop_system_audio_capture');
+
+ // Convert base64 to Blob
+ const binaryString = atob(base64Data);
+ const bytes = new Uint8Array(binaryString.length);
+ for (let i = 0; i < binaryString.length; i++) {
+ bytes[i] = binaryString.charCodeAt(i);
+ }
+
+ const blob = new Blob([bytes], { type: 'audio/wav' });
+ onRecordingComplete?.(blob);
+ } catch (err) {
+ const errorMessage =
+ err instanceof Error
+ ? err.message
+ : 'Failed to stop system audio capture.';
+ setError(errorMessage);
+ }
+ }, [isRecording, onRecordingComplete]);
+
+ // Store stopRecording in ref for use in timer
+ useEffect(() => {
+ stopRecordingRef.current = stopRecording;
+ }, [stopRecording]);
+
+ const cancelRecording = useCallback(async () => {
+ if (isRecording) {
+ await stopRecording();
+ }
+
+ setIsRecording(false);
+ setDuration(0);
+
+ if (timerRef.current !== null) {
+ clearInterval(timerRef.current);
+ timerRef.current = null;
+ }
+ }, [isRecording, stopRecording]);
+
+ // Cleanup on unmount
+ useEffect(() => {
+ return () => {
+ if (timerRef.current !== null) {
+ clearInterval(timerRef.current);
+ }
+ // Cancel recording on unmount if still recording
+ if (isRecording) {
+ void cancelRecording();
+ }
+ };
+ // biome-ignore lint/correctness/useExhaustiveDependencies: cancelRecording is stable
+ }, [isRecording]);
+
+ return {
+ isRecording,
+ duration,
+ error,
+ isSupported,
+ startRecording,
+ stopRecording,
+ cancelRecording,
+ };
+}
diff --git a/tauri/src-tauri/Cargo.lock b/tauri/src-tauri/Cargo.lock
index 525a73d3..6773bedf 100644
--- a/tauri/src-tauri/Cargo.lock
+++ b/tauri/src-tauri/Cargo.lock
@@ -103,6 +103,24 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+[[package]]
+name = "bindgen"
+version = "0.72.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
+dependencies = [
+ "bitflags 2.10.0",
+ "cexpr",
+ "clang-sys",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash",
+ "shlex",
+ "syn 2.0.114",
+]
+
[[package]]
name = "bitflags"
version = "1.3.2"
@@ -267,6 +285,15 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c"
+[[package]]
+name = "cexpr"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
+dependencies = [
+ "nom",
+]
+
[[package]]
name = "cfb"
version = "0.7.3"
@@ -312,6 +339,17 @@ dependencies = [
"windows-link 0.2.1",
]
+[[package]]
+name = "clang-sys"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
+dependencies = [
+ "glob",
+ "libc",
+ "libloading 0.8.9",
+]
+
[[package]]
name = "combine"
version = "4.6.7"
@@ -378,6 +416,15 @@ dependencies = [
"libc",
]
+[[package]]
+name = "coreaudio-sys"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ceec7a6067e62d6f931a2baf6f3a751f4a892595bcec1461a3c94ef9949864b6"
+dependencies = [
+ "bindgen",
+]
+
[[package]]
name = "cpufeatures"
version = "0.2.17"
@@ -646,6 +693,12 @@ version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
+[[package]]
+name = "either"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
+
[[package]]
name = "embed-resource"
version = "3.0.6"
@@ -1212,6 +1265,12 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
+[[package]]
+name = "hound"
+version = "3.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f"
+
[[package]]
name = "html5ever"
version = "0.29.1"
@@ -1534,6 +1593,15 @@ dependencies = [
"once_cell",
]
+[[package]]
+name = "itertools"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
+dependencies = [
+ "either",
+]
+
[[package]]
name = "itoa"
version = "1.0.17"
@@ -1666,7 +1734,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e9ec52138abedcc58dc17a7c6c0c00a2bdb4f3427c7f63fa97fd0d859155caf"
dependencies = [
"gtk-sys",
- "libloading",
+ "libloading 0.7.4",
"once_cell",
]
@@ -1686,6 +1754,16 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "libloading"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
+dependencies = [
+ "cfg-if",
+ "windows-link 0.2.1",
+]
+
[[package]]
name = "libredox"
version = "0.1.12"
@@ -1736,6 +1814,15 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
+[[package]]
+name = "malloc_buf"
+version = "0.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb"
+dependencies = [
+ "libc",
+]
+
[[package]]
name = "markup5ever"
version = "0.14.1"
@@ -1788,6 +1875,12 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
[[package]]
name = "minisign-verify"
version = "0.2.4"
@@ -1878,12 +1971,31 @@ version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
[[package]]
name = "num-conv"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050"
+[[package]]
+name = "num-integer"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
+dependencies = [
+ "num-traits",
+]
+
[[package]]
name = "num-traits"
version = "0.2.19"
@@ -1915,6 +2027,15 @@ dependencies = [
"syn 2.0.114",
]
+[[package]]
+name = "objc"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1"
+dependencies = [
+ "malloc_buf",
+]
+
[[package]]
name = "objc2"
version = "0.6.3"
@@ -3016,6 +3137,12 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+[[package]]
+name = "screencapturekit"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ccf069cb109cf8e01ebdca0d55dfce45dbbf669e8c56ed5c62150b056d3ec9f"
+
[[package]]
name = "selectors"
version = "0.24.0"
@@ -3504,7 +3631,7 @@ dependencies = [
"tao-macros",
"unicode-segmentation",
"url",
- "windows",
+ "windows 0.61.3",
"windows-core 0.61.2",
"windows-version",
"x11-dl",
@@ -3586,7 +3713,7 @@ dependencies = [
"webkit2gtk",
"webview2-com",
"window-vibrancy",
- "windows",
+ "windows 0.61.3",
]
[[package]]
@@ -3784,7 +3911,7 @@ dependencies = [
"url",
"webkit2gtk",
"webview2-com",
- "windows",
+ "windows 0.61.3",
]
[[package]]
@@ -3810,7 +3937,7 @@ dependencies = [
"url",
"webkit2gtk",
"webview2-com",
- "windows",
+ "windows 0.61.3",
"wry",
]
@@ -4358,6 +4485,12 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
name = "voicebox"
version = "0.1.0"
dependencies = [
+ "base64 0.22.1",
+ "core-foundation-sys",
+ "coreaudio-sys",
+ "hound",
+ "objc",
+ "screencapturekit",
"serde",
"serde_json",
"tauri",
@@ -4367,6 +4500,7 @@ dependencies = [
"tauri-plugin-shell",
"tauri-plugin-updater",
"tokio",
+ "wasapi",
]
[[package]]
@@ -4408,6 +4542,19 @@ dependencies = [
"try-lock",
]
+[[package]]
+name = "wasapi"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7834ac561bea8a7413661fdda62180f9e054f99815269cd3572cf7e40d9c3191"
+dependencies = [
+ "log",
+ "num-integer",
+ "thiserror 2.0.18",
+ "windows 0.62.2",
+ "windows-core 0.62.2",
+]
+
[[package]]
name = "wasi"
version = "0.9.0+wasi-snapshot-preview1"
@@ -4582,7 +4729,7 @@ checksum = "7130243a7a5b33c54a444e54842e6a9e133de08b5ad7b5861cd8ed9a6a5bc96a"
dependencies = [
"webview2-com-macros",
"webview2-com-sys",
- "windows",
+ "windows 0.61.3",
"windows-core 0.61.2",
"windows-implement",
"windows-interface",
@@ -4606,7 +4753,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "381336cfffd772377d291702245447a5251a2ffa5bad679c99e61bc48bacbf9c"
dependencies = [
"thiserror 2.0.18",
- "windows",
+ "windows 0.61.3",
"windows-core 0.61.2",
]
@@ -4662,11 +4809,23 @@ version = "0.61.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"
dependencies = [
- "windows-collections",
+ "windows-collections 0.2.0",
"windows-core 0.61.2",
- "windows-future",
+ "windows-future 0.2.1",
"windows-link 0.1.3",
- "windows-numerics",
+ "windows-numerics 0.2.0",
+]
+
+[[package]]
+name = "windows"
+version = "0.62.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580"
+dependencies = [
+ "windows-collections 0.3.2",
+ "windows-core 0.62.2",
+ "windows-future 0.3.2",
+ "windows-numerics 0.3.1",
]
[[package]]
@@ -4678,6 +4837,15 @@ dependencies = [
"windows-core 0.61.2",
]
+[[package]]
+name = "windows-collections"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610"
+dependencies = [
+ "windows-core 0.62.2",
+]
+
[[package]]
name = "windows-core"
version = "0.61.2"
@@ -4712,7 +4880,18 @@ checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"
dependencies = [
"windows-core 0.61.2",
"windows-link 0.1.3",
- "windows-threading",
+ "windows-threading 0.1.0",
+]
+
+[[package]]
+name = "windows-future"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb"
+dependencies = [
+ "windows-core 0.62.2",
+ "windows-link 0.2.1",
+ "windows-threading 0.2.1",
]
[[package]]
@@ -4759,6 +4938,16 @@ dependencies = [
"windows-link 0.1.3",
]
+[[package]]
+name = "windows-numerics"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26"
+dependencies = [
+ "windows-core 0.62.2",
+ "windows-link 0.2.1",
+]
+
[[package]]
name = "windows-result"
version = "0.3.4"
@@ -4897,6 +5086,15 @@ dependencies = [
"windows-link 0.1.3",
]
+[[package]]
+name = "windows-threading"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37"
+dependencies = [
+ "windows-link 0.2.1",
+]
+
[[package]]
name = "windows-version"
version = "0.1.7"
@@ -5123,7 +5321,7 @@ dependencies = [
"webkit2gtk",
"webkit2gtk-sys",
"webview2-com",
- "windows",
+ "windows 0.61.3",
"windows-core 0.61.2",
"windows-version",
"x11-dl",
diff --git a/tauri/src-tauri/Cargo.toml b/tauri/src-tauri/Cargo.toml
index 3b22f7c0..4f484c9d 100644
--- a/tauri/src-tauri/Cargo.toml
+++ b/tauri/src-tauri/Cargo.toml
@@ -20,6 +20,17 @@ tauri-plugin-shell = "2.0"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
tokio = { version = "1", features = ["full"] }
+hound = "3.5"
+base64 = "0.22"
+
+[target.'cfg(target_os = "macos")'.dependencies]
+screencapturekit = { version = "1", features = ["async"] }
+coreaudio-sys = "0.2"
+objc = "0.2"
+core-foundation-sys = "0.8"
+
+[target.'cfg(target_os = "windows")'.dependencies]
+wasapi = "0.22"
[target.'cfg(not(any(target_os = "android", target_os = "ios")))'.dependencies]
tauri-plugin-updater = "2.0"
diff --git a/tauri/src-tauri/Info.plist b/tauri/src-tauri/Info.plist
index 63df5b17..4099dc8b 100644
--- a/tauri/src-tauri/Info.plist
+++ b/tauri/src-tauri/Info.plist
@@ -8,5 +8,7 @@
<string>voicebox</string>
<key>NSMicrophoneUsageDescription</key>
<string>voicebox needs microphone access to record voice samples for voice cloning.</string>
+ <key>NSScreenCaptureUsageDescription</key>
+ <string>Voicebox needs screen capture access to record system audio for voice samples.</string>
</dict>
</plist>
diff --git a/tauri/src-tauri/build.rs b/tauri/src-tauri/build.rs
index d11286b3..69b18c37 100644
--- a/tauri/src-tauri/build.rs
+++ b/tauri/src-tauri/build.rs
@@ -1,6 +1,24 @@
use std::process::Command;
fn main() {
+ // Link Swift runtime libraries for screencapturekit crate
+ #[cfg(target_os = "macos")]
+ {
+ // Add Swift runtime library paths to RPATH
+ println!("cargo:rustc-link-arg=-Wl,-rpath,/usr/lib/swift");
+ println!("cargo:rustc-link-arg=-L/usr/lib/swift");
+
+ // Also try Xcode's Swift libraries
+ if let Ok(output) = Command::new("xcode-select").arg("-p").output() {
+ if output.status.success() {
+ let xcode_path = String::from_utf8_lossy(&output.stdout).trim().to_string();
+ let swift_lib_path = format!("{}/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx", xcode_path);
+ println!("cargo:rustc-link-arg=-Wl,-rpath,{}", swift_lib_path);
+ println!("cargo:rustc-link-arg=-L{}", swift_lib_path);
+ }
+ }
+ }
+
// Compile macOS Liquid Glass icon
#[cfg(target_os = "macos")]
{
diff --git a/tauri/src-tauri/gen/Assets.car b/tauri/src-tauri/gen/Assets.car
index 15d5b61f..9db1fbf0 100644
Binary files a/tauri/src-tauri/gen/Assets.car and b/tauri/src-tauri/gen/Assets.car differ
diff --git a/tauri/src-tauri/src/audio_capture/macos.rs b/tauri/src-tauri/src/audio_capture/macos.rs
new file mode 100644
index 00000000..d929e65a
--- /dev/null
+++ b/tauri/src-tauri/src/audio_capture/macos.rs
@@ -0,0 +1,213 @@
+use crate::audio_capture::AudioCaptureState;
+use base64::{engine::general_purpose, Engine as _};
+use hound::{WavSpec, WavWriter};
+use screencapturekit::{
+ cm::CMSampleBuffer,
+ shareable_content::SCShareableContent,
+ stream::{
+ configuration::SCStreamConfiguration,
+ content_filter::SCContentFilter,
+ output_trait::SCStreamOutputTrait,
+ output_type::SCStreamOutputType,
+ sc_stream::SCStream,
+ },
+};
+use std::io::Cursor;
+use std::sync::{Arc, Mutex};
+use tokio::sync::mpsc;
+
+pub async fn start_capture(
+ state: &AudioCaptureState,
+ max_duration_secs: u32,
+) -> Result<(), String> {
+ // Reset previous samples
+ state.reset();
+
+ // Get shareable content
+ let content = SCShareableContent::get()
+ .map_err(|e| format!("Failed to get shareable content: {}", e))?;
+
+ // Get first display
+ let displays = content.displays();
+ if displays.is_empty() {
+ return Err("No displays available".to_string());
+ }
+ let display = &displays[0];
+
+ // Create content filter for desktop audio
+ let filter = SCContentFilter::create()
+ .with_display(display)
+ .with_excluding_windows(&[])
+ .build();
+
+ // Create stream configuration - audio only
+ let mut config = SCStreamConfiguration::default();
+ config.set_captures_audio(true);
+ config.set_excludes_current_process_audio(false);
+ config.set_sample_rate(48000); // Use i32 directly
+ config.set_channel_count(2); // Use i32 directly
+
+ // Create stream using builder
+ let (tx, mut rx) = mpsc::channel::<()>(1);
+ *state.stop_tx.lock().unwrap() = Some(tx);
+
+ let samples = state.samples.clone();
+ let sample_rate = state.sample_rate.clone();
+ let channels = state.channels.clone();
+
+ // Set sample rate and channels
+ *sample_rate.lock().unwrap() = 48000;
+ *channels.lock().unwrap() = 2;
+
+ // Create output handler struct
+ struct AudioHandler {
+ samples: Arc<Mutex<Vec<f32>>>,
+ }
+
+ impl SCStreamOutputTrait for AudioHandler {
+ fn did_output_sample_buffer(
+ &self,
+ sample: CMSampleBuffer,
+ _type: SCStreamOutputType,
+ ) {
+ if _type == SCStreamOutputType::Audio {
+ if let Ok(audio_samples) = extract_audio_samples(sample) {
+ let mut samples_guard = self.samples.lock().unwrap();
+ samples_guard.extend_from_slice(&audio_samples);
+ }
+ }
+ }
+ }
+
+ let handler = AudioHandler {
+ samples: samples.clone(),
+ };
+
+ // Create stream
+ let mut stream = SCStream::new(&filter, &config);
+
+ // Add output handler for audio (order: handler, then output_type)
+ stream.add_output_handler(handler, SCStreamOutputType::Audio);
+
+ // Store stream reference
+ *state.stream.lock().unwrap() = Some(stream.clone());
+
+ stream.start_capture().map_err(|e| format!("Failed to start capture: {}", e))?;
+
+ // Spawn task to stop after max duration
+ let stream_clone = stream.clone();
+ tokio::spawn(async move {
+ tokio::select! {
+ _ = tokio::time::sleep(tokio::time::Duration::from_secs(max_duration_secs as u64)) => {
+ // Timeout reached
+ }
+ _ = rx.recv() => {
+ // Manual stop
+ }
+ }
+ let _ = stream_clone.stop_capture();
+ });
+
+ Ok(())
+}
+
+pub async fn stop_capture(state: &AudioCaptureState) -> Result<String, String> {
+ // Signal stop
+ if let Some(tx) = state.stop_tx.lock().unwrap().take() {
+ let _ = tx.send(());
+ }
+
+ // Stop stream if still active
+ if let Some(stream) = state.stream.lock().unwrap().take() {
+ let _ = stream.stop_capture();
+ }
+
+ // Wait a bit for capture to stop
+ tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
+
+ // Get samples
+ let samples = state.samples.lock().unwrap().clone();
+ let sample_rate = *state.sample_rate.lock().unwrap();
+ let channels = *state.channels.lock().unwrap();
+
+ if samples.is_empty() {
+ return Err("No audio samples captured".to_string());
+ }
+
+ // Convert to WAV
+ let wav_data = samples_to_wav(&samples, sample_rate, channels)?;
+
+ // Encode to base64
+ let base64_data = general_purpose::STANDARD.encode(&wav_data);
+
+ Ok(base64_data)
+}
+
+pub fn is_supported() -> bool {
+ // ScreenCaptureKit requires macOS 12.3+
+ // Check if we're on a supported version
+ #[cfg(target_os = "macos")]
+ {
+ // Basic check - ScreenCaptureKit should be available on macOS 12.3+
+ true
+ }
+ #[cfg(not(target_os = "macos"))]
+ {
+ false
+ }
+}
+
+fn extract_audio_samples(sample_buffer: CMSampleBuffer) -> Result<Vec<f32>, String> {
+ // Use the crate's built-in method to get audio buffer list
+ let audio_buffer_list = sample_buffer
+ .audio_buffer_list()
+ .ok_or_else(|| "Failed to get audio buffer list".to_string())?;
+
+ let mut samples = Vec::new();
+
+ // Iterate through audio buffers
+ for buffer in audio_buffer_list.iter() {
+ // Get raw bytes and interpret as f32 samples
+ let data_bytes = buffer.data();
+ let num_samples = data_bytes.len() / std::mem::size_of::<f32>();
+
+ if num_samples > 0 {
+ unsafe {
+ // Interpret bytes as f32 samples
+ let data_ptr = data_bytes.as_ptr() as *const f32;
+ let data = std::slice::from_raw_parts(data_ptr, num_samples);
+ samples.extend_from_slice(data);
+ }
+ }
+ }
+
+ Ok(samples)
+}
+
+fn samples_to_wav(samples: &[f32], sample_rate: u32, channels: u16) -> Result<Vec<u8>, String> {
+ let mut buffer = Vec::new();
+ let cursor = Cursor::new(&mut buffer);
+
+ let spec = WavSpec {
+ channels,
+ sample_rate,
+ bits_per_sample: 16,
+ sample_format: hound::SampleFormat::Int,
+ };
+
+ let mut writer = WavWriter::new(cursor, spec)
+ .map_err(|e| format!("Failed to create WAV writer: {}", e))?;
+
+ // Convert f32 samples to i16
+ for sample in samples {
+ let clamped = sample.clamp(-1.0, 1.0);
+ let i16_sample = (clamped * 32767.0) as i16;
+ writer.write_sample(i16_sample)
+ .map_err(|e| format!("Failed to write sample: {}", e))?;
+ }
+
+ writer.finalize()
+ .map_err(|e| format!("Failed to finalize WAV: {}", e))?;
+
+ Ok(buffer)
+}
diff --git a/tauri/src-tauri/src/audio_capture/mod.rs b/tauri/src-tauri/src/audio_capture/mod.rs
new file mode 100644
index 00000000..54f47856
--- /dev/null
+++ b/tauri/src-tauri/src/audio_capture/mod.rs
@@ -0,0 +1,40 @@
+#[cfg(target_os = "macos")]
+mod macos;
+#[cfg(target_os = "windows")]
+mod windows;
+
+#[cfg(target_os = "macos")]
+pub use macos::*;
+#[cfg(target_os = "windows")]
+pub use windows::*;
+
+use std::sync::{Arc, Mutex};
+
+#[cfg(target_os = "macos")]
+use screencapturekit::stream::sc_stream::SCStream;
+
+pub struct AudioCaptureState {
+ pub samples: Arc<Mutex<Vec<f32>>>,
+ pub sample_rate: Arc<Mutex<u32>>,
+ pub channels: Arc<Mutex<u16>>,
+ pub stop_tx: Arc<Mutex<Option<tokio::sync::mpsc::Sender<()>>>>,
+ #[cfg(target_os = "macos")]
+ pub stream: Arc<Mutex<Option<SCStream>>>,
+}
+
+impl AudioCaptureState {
+ pub fn new() -> Self {
+ Self {
+ samples: Arc::new(Mutex::new(Vec::new())),
+ sample_rate: Arc::new(Mutex::new(44100)),
+ channels: Arc::new(Mutex::new(2)),
+ stop_tx: Arc::new(Mutex::new(None)),
+ #[cfg(target_os = "macos")]
+ stream: Arc::new(Mutex::new(None)),
+ }
+ }
+
+ pub fn reset(&self) {
+ *self.samples.lock().unwrap() = Vec::new();
+ }
+}
diff --git a/tauri/src-tauri/src/audio_capture/windows.rs b/tauri/src-tauri/src/audio_capture/windows.rs
new file mode 100644
index 00000000..57766bff
--- /dev/null
+++ b/tauri/src-tauri/src/audio_capture/windows.rs
@@ -0,0 +1,181 @@
+use crate::audio_capture::AudioCaptureState;
+use base64::{engine::general_purpose, Engine as _};
+use hound::{WavSpec, WavWriter};
+use std::io::Cursor;
+use std::sync::{Arc, Mutex};
+use tokio::sync::mpsc;
+use wasapi::*;
+
+pub async fn start_capture(
+ state: &AudioCaptureState,
+ max_duration_secs: u32,
+) -> Result<(), String> {
+ // Reset previous samples
+ state.reset();
+
+ // Get default audio render device for loopback
+ let device = DeviceEnumerator::new()
+ .map_err(|e| format!("Failed to create device enumerator: {}", e))?
+ .get_default_audio_endpoint(&Direction::Render)
+ .map_err(|e| format!("Failed to get default render device: {}", e))?;
+
+ // Create audio client for loopback capture
+ let audio_client = device
+ .get_iaudioclient()
+ .map_err(|e| format!("Failed to get audio client: {}", e))?;
+
+ // Get mix format
+ let mix_format = audio_client
+ .get_mixformat()
+ .map_err(|e| format!("Failed to get mix format: {}", e))?;
+
+ // Set sample rate and channels
+ *state.sample_rate.lock().unwrap() = mix_format.get_samples_per_sec();
+ *state.channels.lock().unwrap() = mix_format.get_nchannels();
+
+ // Initialize audio client for loopback
+ audio_client
+ .initialize_client(
+ &mix_format,
+ 0, // Buffer duration (0 = default)
+ &Direction::Capture,
+ ShareMode::Shared,
+ true, // Loopback mode
+ )
+ .map_err(|e| format!("Failed to initialize audio client: {}", e))?;
+
+ // Get capture client
+ let capture_client = audio_client
+ .get_audiocaptureclient()
+ .map_err(|e| format!("Failed to get capture client: {}", e))?;
+
+ // Start capture
+ audio_client
+ .start_stream()
+ .map_err(|e| format!("Failed to start stream: {}", e))?;
+
+ let samples = state.samples.clone();
+ let stop_tx = state.stop_tx.clone();
+ let (tx, mut rx) = mpsc::channel::<()>(1);
+ *stop_tx.lock().unwrap() = Some(tx);
+
+ // Spawn capture task - move audio_client and capture_client into the task
+ tokio::spawn(async move {
+ loop {
+ tokio::select! {
+ _ = rx.recv() => {
+ break;
+ }
+ _ = tokio::time::sleep(tokio::time::Duration::from_millis(10)) => {
+ // Try to get available data
+ match capture_client.get_available_samples() {
+ Ok(available) => {
+ if available > 0 {
+ match capture_client.get_buffer::<f32>() {
+ Ok((data, flags)) => {
+ if flags.contains(&StreamFlags::SILENT) {
+ // Silent buffer, skip
+ capture_client.release_buffer(available).ok();
+ continue;
+ }
+
+ // Convert samples to f32 and store
+ let mut samples_guard = samples.lock().unwrap();
+ samples_guard.extend_from_slice(data);
+ capture_client.release_buffer(available).ok();
+ }
+ Err(e) => {
+ eprintln!("Error getting buffer: {}", e);
+ }
+ }
+ }
+ }
+ Err(e) => {
+ eprintln!("Error getting available samples: {}", e);
+ }
+ }
+ }
+ }
+ }
+
+ // Stop the stream when done
+ audio_client.stop_stream().ok();
+ });
+
+ // Spawn timeout task
+ let stop_tx_clone = state.stop_tx.clone();
+ tokio::spawn(async move {
+ tokio::time::sleep(tokio::time::Duration::from_secs(max_duration_secs as u64)).await;
+ if let Some(tx) = stop_tx_clone.lock().unwrap().take() {
+ let _ = tx.send(());
+ }
+ });
+
+ Ok(())
+}
+
+pub async fn stop_capture(state: &AudioCaptureState) -> Result<String, String> {
+ // Signal stop
+ if let Some(tx) = state.stop_tx.lock().unwrap().take() {
+ let _ = tx.send(());
+ }
+
+ // Wait a bit for capture to stop
+ tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
+
+ // Get samples
+ let samples = state.samples.lock().unwrap().clone();
+ let sample_rate = *state.sample_rate.lock().unwrap();
+ let channels = *state.channels.lock().unwrap();
+
+ if samples.is_empty() {
+ return Err("No audio samples captured".to_string());
+ }
+
+ // Convert to WAV
+ let wav_data = samples_to_wav(&samples, sample_rate, channels)?;
+
+ // Encode to base64
+ let base64_data = general_purpose::STANDARD.encode(&wav_data);
+
+ Ok(base64_data)
+}
+
+pub fn is_supported() -> bool {
+ #[cfg(target_os = "windows")]
+ {
+ true
+ }
+ #[cfg(not(target_os = "windows"))]
+ {
+ false
+ }
+}
+
+fn samples_to_wav(samples: &[f32], sample_rate: u32, channels: u16) -> Result<Vec<u8>, String> {
+ let mut buffer = Vec::new();
+ let cursor = Cursor::new(&mut buffer);
+
+ let spec = WavSpec {
+ channels,
+ sample_rate,
+ bits_per_sample: 16,
+ sample_format: hound::SampleFormat::Int,
+ };
+
+ let mut writer = WavWriter::new(cursor, spec)
+ .map_err(|e| format!("Failed to create WAV writer: {}", e))?;
+
+ // Convert f32 samples to i16
+ for sample in samples {
+ let clamped = sample.clamp(-1.0, 1.0);
+ let i16_sample = (clamped * 32767.0) as i16;
+ writer.write_sample(i16_sample)
+ .map_err(|e| format!("Failed to write sample: {}", e))?;
+ }
+
+ writer.finalize()
+ .map_err(|e| format!("Failed to finalize WAV: {}", e))?;
+
+ Ok(buffer)
+}
diff --git a/tauri/src-tauri/src/main.rs b/tauri/src-tauri/src/main.rs
index 4af341b8..667ee952 100644
--- a/tauri/src-tauri/src/main.rs
+++ b/tauri/src-tauri/src/main.rs
@@ -1,6 +1,8 @@
// Prevents additional console window on Windows in release, DO NOT REMOVE!!
#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
+mod audio_capture;
+
use std::sync::Mutex;
use tauri::{command, State, Manager, WindowEvent, Emitter, Listener};
use tauri_plugin_shell::ShellExt;
@@ -165,6 +167,26 @@ async fn stop_server(state: State<'_, ServerState>) -> Result<(), String> {
Ok(())
}
+#[command]
+async fn start_system_audio_capture(
+ state: State<'_, audio_capture::AudioCaptureState>,
+ max_duration_secs: u32,
+) -> Result<(), String> {
+ audio_capture::start_capture(&state, max_duration_secs).await
+}
+
+#[command]
+async fn stop_system_audio_capture(
+ state: State<'_, audio_capture::AudioCaptureState>,
+) -> Result<String, String> {
+ audio_capture::stop_capture(&state).await
+}
+
+#[command]
+fn is_system_audio_supported() -> bool {
+ audio_capture::is_supported()
+}
+
#[cfg_attr(mobile, tauri::mobile_entry_point)]
pub fn run() {
tauri::Builder::default()
@@ -174,6 +196,7 @@ pub fn run() {
.manage(ServerState {
child: Mutex::new(None),
})
+ .manage(audio_capture::AudioCaptureState::new())
.setup(|app| {
#[cfg(desktop)]
app.handle().plugin(tauri_plugin_updater::Builder::new().build())?;
@@ -190,7 +213,13 @@ pub fn run() {
}
Ok(())
})
- .invoke_handler(tauri::generate_handler![start_server, stop_server])
+ .invoke_handler(tauri::generate_handler![
+ start_server,
+ stop_server,
+ start_system_audio_capture,
+ stop_system_audio_capture,
+ is_system_audio_supported
+ ])
.on_window_event(|window, event| {
if let WindowEvent::CloseRequested { api, .. } = event {
// Prevent automatic close