Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

11.2 KB
Raw Blame
import { useState } from 'react';
import * as DocumentPicker from 'expo-document-picker';
import * as FileSystem from 'expo-file-system/legacy';
import { AudioTools } from '../../modules/audio-tools/src';
import {
  isOggOpus,
  isLocalModelMissingError,
} from '../services/transcription/TranscriptionService';
import { createTranscriptId, useTranscriptStore } from '../store/useTranscriptStore';
import { snapshotTextInference, snapshotTranscriptionJob } from '../lib/inferenceRouting';
import { transcribeAndCleanup } from '../lib/transcribeAndCleanup';
import { getPreferredTranscriptionLanguage } from '../lib/transcriptionLanguage';
import { toFriendlyTranscriptionErrorMessage } from '../lib/transcriptionErrors';
import { isNoSpeechError } from '../lib/permissions';
import { deleteManagedTranscriptAudio, retainTranscriptAudio } from '../lib/transcriptAudio';
import { isUsageLimitError, type UsageLimitError } from '../lib/usageLimitError';
import { randomUUID } from '../lib/uuid';
import { MAX_FILE_SIZE } from '../config/constants';
import type { TranscriptionProvider } from '../types';

export interface UseFileUploadOptions {
  onComplete?: (text: string) => void;
  onError?: (error: Error) => void;
  // Private mode but the on-device model is unavailable. We never upload
  // silently — the caller prompts, and may invoke retryWithCloud to transcribe
  // the picked file in the cloud instead.
  onLocalModelMissing?: (retryWithCloud: () => void) => void;
  // Cloud quota is exhausted. The caller can present billing, refresh backend
  // usage, then invoke retryWithCloud only if quota/subscription now permits it.
  onUsageLimitReached?: (
    error: UsageLimitError,
    retryWithCloud: () => Promise<void>,
  ) => void | Promise<void>;
}

const MAX_FILE_SIZE_MB = Math.round(MAX_FILE_SIZE / (1024 * 1024));

// OpenAI-compatible transcription endpoints pick the decoder from the file
// extension. A file the provider rejects would become a failed row whose retry
// (pinned to the same provider) can never succeed, so it is refused up front.
const PROVIDER_AUDIO_EXTENSIONS = new Set([
  'flac',
  'mp3',
  'mp4',
  'mpeg',
  'mpga',
  'm4a',
  'ogg',
  'wav',
  'webm',
]);

async function assertProviderAcceptsFile(
  file: DocumentPicker.DocumentPickerAsset,
  extension: string | undefined,
): Promise<void> {
  if (!extension || !PROVIDER_AUDIO_EXTENSIONS.has(extension)) {
    throw new Error(
      'Your provider accepts FLAC, MP3, MP4, M4A, OGG, WAV, or WEBM audio. Choose a file in one of those formats.',
    );
  }
  // The picker does not always report a size; the provider's limit is hard.
  const size =
    file.size ??
    (await FileSystem.getInfoAsync(file.uri)
      .then((info) => (info.exists ? info.size : undefined))
      .catch(() => undefined));
  if (size !== undefined && size > MAX_FILE_SIZE) {
    throw new Error(
      `Your provider accepts audio files up to ${MAX_FILE_SIZE_MB} MB. Choose a smaller file.`,
    );
  }
}

// The native module rejects unreadable/unsupported audio with these codes;
// surface a clear instruction instead of the raw AVFoundation error.
const toFriendlyUploadError = (error: unknown): Error => {
  const code = (error as { code?: string })?.code;
  if (code === 'AUDIO_TRANSCODE_ERROR' || code === 'AUDIO_CHUNK_ERROR') {
    return new Error("Couldn't read this audio file. Please try a WAV, MP3, or M4A file.");
  }
  return error instanceof Error ? error : new Error(String(error));
};

export function useFileUpload(options: UseFileUploadOptions = {}) {
  const [isProcessing, setIsProcessing] = useState(false);
  const [currentText, setCurrentText] = useState('');
  const addTranscript = useTranscriptStore((state) => state.addTranscript);
  const addFailedTranscript = useTranscriptStore((state) => state.addFailedTranscript);

  const pickAndTranscribeFile = async () => {
    try {
      const allowedMimeTypes = new Set([
        'audio/wav',
        'audio/x-wav',
        'audio/wave',
        'audio/mpeg',
        'audio/mp3',
        'audio/mp4',
        'audio/m4a',
        'audio/aac',
        'audio/ogg',
        'audio/opus',
        'audio/webm',
      ]);
      const allowedExtensions = new Set(['wav', 'mp3', 'm4a', 'aac', 'ogg', 'opus', 'webm']);

      const result = await DocumentPicker.getDocumentAsync({
        type: '*/*',
        copyToCacheDirectory: true,
      });

      if (result.canceled) {
        return;
      }

      let jobRoute = snapshotTranscriptionJob('upload');
      const transcriptionProvider = jobRoute.provider;
      const file = result.assets[0];

      const extension = file.name?.includes('.')
        ? file.name.split('.').pop()?.toLowerCase()
        : undefined;
      const isMimeAllowed = file.mimeType ? allowedMimeTypes.has(file.mimeType) : false;
      const isExtensionAllowed = extension ? allowedExtensions.has(extension) : false;

      if (!isMimeAllowed && !isExtensionAllowed) {
        throw new Error(
          'Unsupported file type. Please select a WAV, MP3, M4A, AAC, OGG/Opus, or WEBM audio file.',
        );
      }

      if (!file.uri) {
        throw new Error('No file selected');
      }

      if (file.size && file.size > MAX_FILE_SIZE) {
        throw new Error(
          `File is too large. Please select an audio file under ${MAX_FILE_SIZE_MB}MB.`,
        );
      }

      if (transcriptionProvider === 'byok') {
        await assertProviderAcceptsFile(file, extension);
      }

      // On-device Opus decoding isn't supported yet; the cloud transcriber
      // decodes Opus, so route private-mode Opus uploads there instead.
      if (transcriptionProvider === 'local' && isOggOpus(file.name, file.mimeType, file.uri)) {
        throw new Error(
          "Opus audio can't be transcribed on this iPhone yet. Set Uploads to OpenWhispr Cloud in Account → AI Models, then try again.",
        );
      }

      setCurrentText('');
      const transcriptId = createTranscriptId();
      const clientTranscriptionId = `file-${randomUUID()}`;
      const retainedAudioUrl = await retainTranscriptAudio(file.uri, {
        transcriptId,
        audioFileName: file.name,
        audioMimeType: file.mimeType,
      });

      const addFailedUpload = async (provider: TranscriptionProvider, error: unknown) => {
        if (isNoSpeechError(error)) {
          await deleteManagedTranscriptAudio(retainedAudioUrl);
          return;
        }
        try {
          await addFailedTranscript({
            id: transcriptId,
            audioUrl: retainedAudioUrl,
            audioFileName: file.name,
            audioMimeType: file.mimeType,
            provider,
            inferenceRoute: provider === 'byok' ? jobRoute.inferenceRoute : undefined,
            cleanupRoute: jobRoute.cleanupRoute,
            cleanupUnavailable: jobRoute.cleanupUnavailable,
            requestContext: 'file',
            jobId: transcriptId,
            errorMessage: toFriendlyTranscriptionErrorMessage(error),
          });
        } catch (failedRowError) {
          if (__DEV__) {
            console.warn('[upload] Failed to save retryable failed transcript:', failedRowError);
          }
        }
      };

      // whisper.rn only reads 16kHz mono WAV, so compressed uploads must be
      // transcoded before local transcription. Cloud sends the original file.
      const runTranscription = async (provider: TranscriptionProvider) => {
        let audioUri = retainedAudioUrl;
        const tempUris: string[] = [];
        try {
          if (provider === 'local' && AudioTools.isAvailable()) {
            const transcoded = await AudioTools.transcodeToWav(retainedAudioUrl);
            audioUri = transcoded.uri;
            tempUris.push(transcoded.uri);
          }

          const processedResult = await transcribeAndCleanup(
            {
              ...jobRoute,
              audioUri,
              provider,
              inferenceRoute: provider === 'byok' ? jobRoute.inferenceRoute : undefined,
              cleanupRoute: jobRoute.cleanupRoute,
              cleanupUnavailable: jobRoute.cleanupUnavailable,
              fileName: file.name,
              mimeType: file.mimeType,
              language: getPreferredTranscriptionLanguage(),
              requestContext: 'file',
              jobId: transcriptId,
              clientTranscriptionId,
            },
            {
              onRawTranscript: (text) => setCurrentText(text),
            },
          );
          const finalText = processedResult.text;

          await addTranscript({
            id: transcriptId,
            text: finalText,
            originalText: processedResult.originalText,
            audioUrl: retainedAudioUrl,
            audioFileName: file.name,
            audioMimeType: file.mimeType,
            duration: processedResult.transcription.duration,
            provider: processedResult.transcription.provider,
            inferenceRoute: processedResult.transcription.inferenceRoute,
            cleanupRoute: processedResult.transcription.cleanupRoute,
            cleanupUnavailable: processedResult.transcription.cleanupUnavailable,
            cleanupWarning: processedResult.transcription.cleanupWarning,
            requestContext: 'file',
            jobId: transcriptId,
          });

          setCurrentText(finalText);
          options.onComplete?.(finalText);
        } finally {
          await AudioTools.cleanup(tempUris);
        }
      };

      const retryWithCloud = async () => {
        setIsProcessing(true);
        try {
          jobRoute = { provider: 'cloud', ...snapshotTextInference('cloud') };
          await runTranscription('cloud');
        } catch (retryError) {
          if (isUsageLimitError(retryError) && options.onUsageLimitReached) {
            await options.onUsageLimitReached(retryError, retryWithCloud);
            return;
          }
          addFailedUpload('cloud', retryError).catch(() => undefined);
          options.onError?.(toFriendlyUploadError(retryError));
        } finally {
          setIsProcessing(false);
        }
      };

      setIsProcessing(true);
      try {
        await runTranscription(transcriptionProvider);
      } catch (error) {
        // Private mode with no on-device model. Don't upload silently — let the
        // caller prompt, then transcribe the picked file in the cloud if approved.
        if (
          transcriptionProvider === 'local' &&
          isLocalModelMissingError(error) &&
          options.onLocalModelMissing
        ) {
          await addFailedUpload(transcriptionProvider, error);
          options.onLocalModelMissing(retryWithCloud);
          return;
        }
        // Cloud quota exhausted. Persist the retryable upload, then let the
        // caller present billing and re-run the cloud transcription if approved.
        if (isUsageLimitError(error) && options.onUsageLimitReached) {
          await addFailedUpload(transcriptionProvider, error);
          await options.onUsageLimitReached(error, retryWithCloud);
          return;
        }
        await addFailedUpload(transcriptionProvider, error);
        throw error;
      }
    } catch (error) {
      const friendly = toFriendlyUploadError(error);
      options.onError?.(friendly);
      throw friendly;
    } finally {
      setIsProcessing(false);
    }
  };

  return {
    isProcessing,
    currentText,
    pickAndTranscribeFile,
  };
}