Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

19.4 KB
Raw Blame
import { useRef, useState } from 'react';
import { Platform } from 'react-native';
import * as FileSystem from 'expo-file-system/legacy';
import { createTranscriptId, useTranscriptStore } from '../store/useTranscriptStore';
import {
  snapshotTextInference,
  snapshotTranscriptionJob,
  type TranscriptionJobRoute,
} from '../lib/inferenceRouting';
import { transcribeAndCleanup } from '../lib/transcribeAndCleanup';
import { isLocalModelMissingError } from '../services/transcription/TranscriptionService';
import { analyzeSpeechActivity, createNoSpeechError } from '../lib/speechActivity';
import {
  isMicPermissionError,
  isNoSpeechError,
  MicPermissionError,
  NO_SPEECH_ERROR_MESSAGE,
} from '../lib/permissions';
import { getPreferredTranscriptionLanguage } from '../lib/transcriptionLanguage';
import { toFriendlyTranscriptionErrorMessage } from '../lib/transcriptionErrors';
import { deleteManagedTranscriptAudio, retainTranscriptAudio } from '../lib/transcriptAudio';
import { isUsageLimitError, type UsageLimitError } from '../lib/usageLimitError';
import { randomUUID } from '../lib/uuid';
import type { TranscriptionProvider } from '../types';
import {
  getExpoAudioModule,
  isExpoAudioAvailable,
  getDefaultRecorderOptions,
} from '../utils/expoAudio';
import { useAudioRecorder } from 'expo-audio';

export interface UseAudioRecordingOptions {
  onComplete?: (text: string) => void;
  onError?: (error: Error) => void;
  allowsBackgroundRecording?: boolean;
  // Defaults to true. When false, nothing reaches transcript history (no row on
  // success or failure) and the recording is deleted once its outcome is known.
  saveToHistory?: boolean;
  // Private mode but the on-device model is unavailable. We never fall back to
  // the cloud silently — the caller prompts, and may invoke retryWithCloud to
  // transcribe the recording we already captured. Call discard when the user
  // declines; it deletes the recording only when saveToHistory is false, since
  // otherwise the failed history row keeps it for a later retry.
  onLocalModelMissing?: (retryWithCloud: () => void, discard: () => Promise<void>) => void;
  // Cloud quota is exhausted. The caller can present billing, refresh backend
  // usage, then invoke retryWithCloud only if quota/subscription now permits it.
  onUsageLimitReached?: (
    error: UsageLimitError,
    retryWithCloud: () => Promise<void>,
  ) => void | Promise<void>;
  // Defaults to true. When false the recording gets its raw transcript back: no cleanup pass,
  // fused or separate, and no snippet expansion, so nothing rewrites it or takes it to a cleanup
  // route its transcription never used.
  cleanup?: boolean;
  // Polled from the moment the recording stops. Once it returns true, transcription retries and
  // cleanup stop and the recording is discarded without reaching history, onComplete or onError.
  shouldCancel?: () => boolean;
}

type RetainedRecordingAudio = {
  id: string;
  audioUrl: string;
  audioFileName?: string;
  audioMimeType?: string;
};

// Marks cleanup unavailable for a recording without cleanup: the job then skips the fused cloud
// cleanup and the separate pass alike, even with the cleanup route it fills in for itself.
const CLEANUP_OFF = 'Cleanup is off for this recording.';

const fileNameFromUri = (uri: string): string | undefined => uri.split('/').pop() || undefined;

const mimeTypeFromFileName = (fileName?: string): string | undefined => {
  const extension = fileName?.split('.').pop()?.toLowerCase();
  if (extension === 'wav' || extension === 'wave') return 'audio/wav';
  if (extension === 'm4a') return 'audio/m4a';
  if (extension === 'mp3') return 'audio/mpeg';
  return undefined;
};

export function useAudioRecording(options: UseAudioRecordingOptions = {}) {
  const [isRecording, setIsRecording] = useState(false);
  const [isProcessing, setIsProcessing] = useState(false);
  const [currentText, setCurrentText] = useState('');

  // Import expo-audio module
  const expoAudio = getExpoAudioModule();
  const addTranscript = useTranscriptStore((state) => state.addTranscript);
  const addFailedTranscript = useTranscriptStore((state) => state.addFailedTranscript);
  const jobRouteRef = useRef<TranscriptionJobRoute>({ provider: 'cloud' });
  // Held only across a native stop: whichever stop or cancel claims it owns the recording, so a
  // second one never stops it again or deletes the file the first is transcribing.
  const stoppingRef = useRef(false);
  // Whether the recording this hook started is still open. A call or Siri pauses the recorder,
  // which then reports not recording, so cancel can't read that from the recorder.
  const openRef = useRef(false);

  const recorderOptions = getDefaultRecorderOptions();
  const audioRecorder = useAudioRecorder(recorderOptions);
  const isSupported = isExpoAudioAvailable() && !!audioRecorder;
  const allowsBackgroundRecording = options.allowsBackgroundRecording === true;
  const saveToHistory = options.saveToHistory !== false;
  const cleanup = options.cleanup !== false;
  const isCancelled = (): boolean => options.shouldCancel?.() === true;

  const ensureAudioSupport = () => {
    if (!isSupported || !expoAudio || !audioRecorder) {
      const error = new Error(
        'Audio recording is unavailable in Expo Go. Build and run a development client (e.g. `expo run:ios`) to test recording features.',
      );
      if (__DEV__) {
        console.warn('[audio] Attempted to use recording without native module support.');
      }
      throw error;
    }
    return { expoAudio, audioRecorder };
  };

  const startRecording = async () => {
    try {
      const { expoAudio: audioModule, audioRecorder: recorder } = ensureAudioSupport();

      // Check status before requesting: iOS shows its system dialog only on
      // the very first-ever request for this install. If canAskAgain is
      // already false here, the upcoming request call is guaranteed to
      // return silently with no native UI.
      const priorStatus = await audioModule.AudioModule.getRecordingPermissionsAsync();
      const nativePromptShown = !priorStatus.granted && priorStatus.canAskAgain;

      const { granted, canAskAgain } =
        await audioModule.AudioModule.requestRecordingPermissionsAsync();
      if (!granted) {
        throw new MicPermissionError('Microphone permission not granted', {
          canAskAgain,
          nativePromptShown,
        });
      }

      if (Platform.OS === 'android' && allowsBackgroundRecording) {
        const notificationPermission =
          await audioModule.AudioModule.requestNotificationPermissionsAsync();
        if (!notificationPermission.granted) {
          throw new Error(
            'Notification permission is required to keep recording in the background.',
          );
        }
      }

      try {
        await audioModule.AudioModule.setAudioModeAsync({
          allowsRecording: true,
          playsInSilentMode: true,
          interruptionMode: 'mixWithOthers',
          shouldPlayInBackground: allowsBackgroundRecording,
          shouldRouteThroughEarpiece: false,
          allowsBackgroundRecording,
        });
      } catch (audioModeError) {
        console.warn('[audio] Failed to set audio mode:', audioModeError);
      }

      // Release any stale state from a previous recording attempt.
      if (recorder.isRecording) {
        try {
          await recorder.stop();
        } catch {}
      }

      jobRouteRef.current = cleanup
        ? snapshotTranscriptionJob('dictation')
        : { ...snapshotTranscriptionJob('dictation', []), cleanupUnavailable: CLEANUP_OFF };
      await recorder.prepareToRecordAsync(recorderOptions);
      await recorder.record();
      openRef.current = true;

      setIsRecording(true);
      setCurrentText('');
    } catch (error) {
      // Permission denial is a normal user state — surfaced by onError as a
      // friendly alert. Skip console.error so dev LogBox doesn't show a red toast.
      if (!isMicPermissionError(error)) {
        console.error('[audio] Start recording error:', error);
      }
      if (error instanceof Error) {
        options.onError?.(error);
      }
      throw error;
    }
  };

  const stopRecording = async () => {
    let sourceUriToDiscard: string | null = null;
    try {
      const { audioRecorder: recorder } = ensureAudioSupport();

      if (!isRecording || stoppingRef.current) {
        return;
      }

      stoppingRef.current = true;
      try {
        await recorder.stop();
        openRef.current = false;
      } finally {
        stoppingRef.current = false;
      }

      const uri = recorder.uri;
      if (!saveToHistory) sourceUriToDiscard = uri ?? null;

      setIsRecording(false);
      setIsProcessing(true);

      if (!uri) {
        throw new Error('No recording URI - recorder may not have started properly');
      }

      const fileInfo = await FileSystem.getInfoAsync(uri).catch((error) => {
        console.warn('[audio] Could not verify recording file:', error);
        return null;
      });

      if (fileInfo) {
        if (!fileInfo.exists) {
          throw new Error(
            'Recording file was not created. Audio recording may not work in the iOS Simulator. Please try on a physical device.',
          );
        }
        if (fileInfo.size === 0) {
          throw new Error(
            'Recording file is empty. No audio was captured. This may be a simulator limitation.',
          );
        }
      }

      const speechAnalysis = await analyzeSpeechActivity(uri, 'recording');
      if (speechAnalysis?.noSpeechLikely) {
        throw createNoSpeechError();
      }
      const clientTranscriptionId = `recording-${randomUUID()}`;

      const retained = await retainRecording(uri);
      sourceUriToDiscard = null;
      const transcriptionProvider = jobRouteRef.current.provider;
      try {
        await finalizeRecording(retained, transcriptionProvider, clientTranscriptionId);
      } catch (transcribeError) {
        if (isCancelled()) {
          await discardRetainedAudio(retained);
          return;
        }
        if (isNoSpeechError(transcribeError)) {
          await deleteManagedTranscriptAudio(retained.audioUrl);
          throw new Error(NO_SPEECH_ERROR_MESSAGE);
        }
        // Private mode with no on-device model. Never upload silently — hand the
        // choice to the caller. The recording is kept so an approved cloud retry
        // can reuse it without re-recording.
        if (
          transcriptionProvider === 'local' &&
          isLocalModelMissingError(transcribeError) &&
          options.onLocalModelMissing
        ) {
          if (saveToHistory) {
            await addFailedRecording(retained, transcriptionProvider, transcribeError);
          }
          options.onLocalModelMissing(
            () => {
              retryWithCloud(retained, clientTranscriptionId);
            },
            () => (saveToHistory ? Promise.resolve() : discardRetainedAudio(retained)),
          );
          return;
        }
        // Cloud quota exhausted. Keep the recording so an approved retry can
        // reuse it, then hand the billing/retry decision to the caller.
        if (isUsageLimitError(transcribeError) && options.onUsageLimitReached) {
          if (saveToHistory) {
            await addFailedRecording(retained, transcriptionProvider, transcribeError);
          }
          await options.onUsageLimitReached(transcribeError, () =>
            retryWithCloud(retained, clientTranscriptionId),
          );
          // Recovery is over, retried or declined; a retry already removed it.
          if (!saveToHistory) await discardRetainedAudio(retained);
          return;
        }
        if (saveToHistory) {
          await addFailedRecording(retained, transcriptionProvider, transcribeError);
        } else {
          await discardRetainedAudio(retained);
        }
        throw transcribeError;
      }
    } catch (error) {
      if (isCancelled()) return;
      options.onError?.(error as Error);
      throw error;
    } finally {
      // Retention normally removes the source. An ephemeral question must also lose it when
      // validation or copying fails before managed audio can take ownership.
      if (sourceUriToDiscard) {
        await FileSystem.deleteAsync(sourceUriToDiscard, { idempotent: true }).catch(
          () => undefined,
        );
      }
      setIsProcessing(false);
    }
  };

  const retainRecording = async (uri: string): Promise<RetainedRecordingAudio> => {
    const id = createTranscriptId();
    const audioFileName = fileNameFromUri(uri);
    const audioMimeType = mimeTypeFromFileName(audioFileName);
    const audioUrl = await retainTranscriptAudio(uri, {
      transcriptId: id,
      audioFileName,
      audioMimeType,
    });
    return { id, audioUrl, audioFileName, audioMimeType };
  };

  // Never lets a cleanup failure replace the result or error being reported.
  const discardRetainedAudio = async (retained: RetainedRecordingAudio): Promise<void> => {
    await deleteManagedTranscriptAudio(retained.audioUrl).catch(() => undefined);
  };

  const addFailedRecording = async (
    retained: RetainedRecordingAudio,
    provider: TranscriptionProvider,
    error: unknown,
  ) => {
    try {
      await addFailedTranscript({
        id: retained.id,
        audioUrl: retained.audioUrl,
        audioFileName: retained.audioFileName,
        audioMimeType: retained.audioMimeType,
        provider,
        inferenceRoute: provider === 'byok' ? jobRouteRef.current.inferenceRoute : undefined,
        cleanupRoute: jobRouteRef.current.cleanupRoute,
        cleanupUnavailable: jobRouteRef.current.cleanupUnavailable,
        requestContext: 'recording',
        jobId: retained.id,
        errorMessage: toFriendlyTranscriptionErrorMessage(error),
      });
    } catch (failedRowError) {
      if (__DEV__) {
        console.warn('[audio] Failed to save retryable failed transcript:', failedRowError);
      }
    }
  };

  const finalizeRecording = async (
    retained: RetainedRecordingAudio,
    provider: TranscriptionProvider,
    clientTranscriptionId: string,
  ) => {
    if (isCancelled()) {
      await discardRetainedAudio(retained);
      return;
    }
    const processedResult = await transcribeAndCleanup(
      {
        ...jobRouteRef.current,
        audioUri: retained.audioUrl,
        provider,
        inferenceRoute: provider === 'byok' ? jobRouteRef.current.inferenceRoute : undefined,
        cleanupRoute: jobRouteRef.current.cleanupRoute,
        cleanupUnavailable: jobRouteRef.current.cleanupUnavailable,
        language: getPreferredTranscriptionLanguage(),
        fileName: retained.audioFileName,
        mimeType: retained.audioMimeType,
        requestContext: 'recording',
        jobId: retained.id,
        clientTranscriptionId,
      },
      {
        onRawTranscript: (text) => setCurrentText(text),
        shouldCancel: isCancelled,
      },
    );

    if (isCancelled()) {
      await discardRetainedAudio(retained);
      return;
    }
    // Snippets expand only the final text; originalText stays the transcript as heard.
    const finalText = cleanup ? processedResult.text : processedResult.originalText;
    if (!finalText.trim()) {
      throw new Error(NO_SPEECH_ERROR_MESSAGE);
    }

    if (!saveToHistory) {
      await discardRetainedAudio(retained);
      setCurrentText(finalText);
      options.onComplete?.(finalText);
      return;
    }

    // The transcript is already paid for, so a history write failure must not lose it.
    const saved = await addTranscript({
      id: retained.id,
      text: finalText,
      originalText: processedResult.originalText,
      audioUrl: retained.audioUrl,
      audioFileName: retained.audioFileName,
      audioMimeType: retained.audioMimeType,
      duration: processedResult.transcription.duration,
      provider: processedResult.transcription.provider,
      inferenceRoute: processedResult.transcription.inferenceRoute,
      cleanupRoute: processedResult.transcription.cleanupRoute,
      cleanupUnavailable: processedResult.transcription.cleanupUnavailable,
      cleanupWarning: processedResult.transcription.cleanupWarning,
      requestContext: 'recording',
      jobId: retained.id,
    }).then(
      () => true,
      () => false,
    );

    setCurrentText(finalText);
    options.onComplete?.(finalText);
    if (!saved) options.onError?.(new Error('Your transcript could not be saved to history.'));
  };

  // Consented cloud transcription of an already-captured private-mode recording.
  const retryWithCloud = async (
    retained: RetainedRecordingAudio,
    clientTranscriptionId: string,
  ) => {
    setIsProcessing(true);
    try {
      // The private-mode snapshot pinned local text stages; a consented Cloud
      // upload should clean the way a Cloud recording would.
      jobRouteRef.current = cleanup
        ? { provider: 'cloud', ...snapshotTextInference('cloud') }
        : { provider: 'cloud', cleanupUnavailable: CLEANUP_OFF };
      await finalizeRecording(retained, 'cloud', clientTranscriptionId);
    } catch (error) {
      if (isCancelled()) {
        await discardRetainedAudio(retained);
        return;
      }
      if (isUsageLimitError(error) && options.onUsageLimitReached) {
        await options.onUsageLimitReached(error, () =>
          retryWithCloud(retained, clientTranscriptionId),
        );
        if (!saveToHistory) await discardRetainedAudio(retained);
        return;
      }
      if (!saveToHistory) {
        await discardRetainedAudio(retained);
      } else if (!isNoSpeechError(error)) {
        await addFailedRecording(retained, 'cloud', error);
      }
      options.onError?.(
        isNoSpeechError(error) ? new Error(NO_SPEECH_ERROR_MESSAGE) : (error as Error),
      );
    } finally {
      setIsProcessing(false);
    }
  };

  // A cancelled recording is discarded, so the file it was writing goes too. A recording another
  // stop already claimed is left to it, file and processing state alike. The claim is held until
  // the recording is gone, so a send landing meanwhile can't stop it again and read the deleted file.
  const cancelRecording = async (): Promise<void> => {
    if (stoppingRef.current) return;
    if (!audioRecorder || (!audioRecorder.isRecording && !openRef.current)) {
      setIsRecording(false);
      return;
    }
    stoppingRef.current = true;
    // Read before stopping: the recorder may move on to a new file once it stops.
    const uri = audioRecorder.uri;
    try {
      const paused = !audioRecorder.isRecording;
      try {
        // A paused recorder ignores cancel; stopping it finishes the file so it can be deleted.
        await audioRecorder.stop();
      } catch (stopError) {
        if (!paused) throw stopError;
      }
      openRef.current = false;
      if (uri) {
        await FileSystem.deleteAsync(uri, { idempotent: true }).catch(() => undefined);
      }
      setIsRecording(false);
      setIsProcessing(false);
      setCurrentText('');
    } finally {
      stoppingRef.current = false;
    }
  };

  // Meeting-specific: stops recording and returns the raw URI without running the dictation finalize path.
  const stopRecordingRaw = async (): Promise<string | null> => {
    if (!audioRecorder.isRecording) return null;
    stoppingRef.current = true;
    try {
      await audioRecorder.stop();
      openRef.current = false;
    } finally {
      stoppingRef.current = false;
    }
    setIsRecording(false);
    setIsProcessing(false);
    return audioRecorder.uri ?? null;
  };

  return {
    isRecording,
    isProcessing,
    currentText,
    isSupported,
    startRecording,
    stopRecording,
    stopRecordingRaw,
    cancelRecording,
    audioRecorder,
  };
}