Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

20.4 KB
Raw Blame
import { isProviderJobActive } from '@/lib/providerJobActivity';
import { AppGroupStorage } from '../../modules/app-group-storage/src';
import { create } from 'zustand';
import {
  listPendingProviderRecoveryJobs,
  clearKeyboardProviderRecovery,
} from '@/lib/keyboardInferenceRoute';
import { AudioTools } from '../../modules/audio-tools/src';
import { transcribeAndCleanup } from '@/lib/transcribeAndCleanup';
import { getPreferredTranscriptionLanguage } from '@/lib/transcriptionLanguage';
import { toFriendlyTranscriptionErrorMessage } from '@/lib/transcriptionErrors';
import {
  deleteManagedTranscriptAudio,
  garbageCollectTranscriptAudio,
  isManagedTranscriptAudioUri,
  isWavAudioFile,
  retainTranscriptAudio,
  RETAINED_AUDIO_MAX_AGE_MS,
  transcriptAudioExists,
} from '@/lib/transcriptAudio';
import { useProcessingModeStore } from '@/store/useProcessingModeStore';
import { snapshotTextInference, snapshotTranscriptionJob } from '@/lib/inferenceRouting';
import { logTranscriptionCompleted } from '@/lib/appsflyer';
import type { KeyboardTone, Transcript, TranscriptionProvider } from '../types';
import { StorageService } from '../services/storage/StorageService';

export type AddTranscriptInput = Omit<Transcript, 'id' | 'createdAt' | 'updatedAt'> & {
  id?: string;
};

export type FailedTranscriptInput = {
  id?: string;
  audioUrl: string;
  audioFileName?: string;
  audioMimeType?: string;
  duration?: number;
  provider: TranscriptionProvider;
  inferenceRoute?: Transcript['inferenceRoute'];
  cleanupRoute?: Transcript['cleanupRoute'];
  cleanupUnavailable?: string;
  requestContext?: Transcript['requestContext'];
  keyboardTone?: KeyboardTone;
  jobId?: string;
  errorMessage: string;
};

interface TranscriptState {
  transcripts: Transcript[];
  currentTranscript: Transcript | null;
  isLoading: boolean;
  // True once the stored history has been read. Every write persists the whole
  // list, so a write before the first read would replace history with one row.
  isLoaded: boolean;
  error: string | null;

  loadTranscripts: () => Promise<void>;
  addTranscript: (transcript: AddTranscriptInput) => Promise<void>;
  addFailedTranscript: (transcript: FailedTranscriptInput) => Promise<void>;
  retryTranscript: (id: string) => Promise<Transcript>;
  updateTranscript: (id: string, updates: Partial<Transcript>) => Promise<void>;
  deleteTranscript: (id: string) => Promise<void>;
  setCurrentTranscript: (transcript: Transcript | null) => void;
  clearTranscripts: () => Promise<void>;
}

export function createTranscriptId(): string {
  return `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
}

const normalizeTranscript = (transcript: Transcript): Transcript => ({
  ...transcript,
  status: transcript.status ?? 'completed',
  retryCount: transcript.retryCount ?? 0,
});

const ensureRetainedAudio = async (
  transcriptId: string,
  audioUrl?: string,
  audioFileName?: string,
  audioMimeType?: string,
): Promise<string | undefined> => {
  if (!audioUrl) return undefined;
  if (isManagedTranscriptAudioUri(audioUrl)) return audioUrl;
  return retainTranscriptAudio(audioUrl, { transcriptId, audioFileName, audioMimeType });
};

const RETRY_JOB_ID = /^(.+)-retry-\d+$/;

// Shared so a write that has to wait for history and the app's own startup load
// read storage once.
let inFlightLoad: Promise<void> | null = null;
// History reads and the storage step of each write run one at a time, so a reload
// can't collect audio a write is about to point at or replace a list it just saved.
let historyQueue: Promise<unknown> = Promise.resolve();
function exclusive<T>(operation: () => Promise<T>): Promise<T> {
  const run = historyQueue.then(operation, operation);
  historyQueue = run.catch(() => undefined);
  return run;
}

export const useTranscriptStore = create<TranscriptState>((set, get) => {
  const ensureLoaded = async (): Promise<void> => {
    if (!get().isLoaded) await get().loadTranscripts();
    if (!get().isLoaded) throw new Error('Transcript history is unavailable. Try again.');
  };

  const readHistory = async (): Promise<void> => {
    set({ isLoading: true, error: null });
    try {
      const now = Date.now();
      const loaded = (await StorageService.getTranscripts()).map(normalizeTranscript);
      const pendingJobs = listPendingProviderRecoveryJobs();
      const activeKeyboardJobId = AppGroupStorage.getItem('keyboard_recording_job_id');
      const recoveredJobIds: string[] = [];
      for (const job of pendingJobs) {
        // The keyboard handoff owns insertion and cleanup for its active job.
        if (
          job.jobId === activeKeyboardJobId ||
          isProviderJobActive(job.jobId) ||
          job.error ||
          !job.route
        )
          continue;
        // A retry saves its recovery entry as `<job id>-retry-<ms>`. One killed
        // mid-request belongs to the row it was retrying, which still has its audio.
        const retriedKey = RETRY_JOB_ID.exec(job.jobId)?.[1];
        const retriedIndex = retriedKey
          ? loaded.findIndex((row) => row.id === retriedKey || row.jobId === retriedKey)
          : -1;
        if (retriedIndex >= 0) {
          const retried = loaded[retriedIndex];
          if (job.result) {
            loaded[retriedIndex] = {
              ...retried,
              ...job.route,
              text: job.result.text,
              originalText: job.result.text,
              updatedAt: now,
              status: 'completed',
              audioUrl: undefined,
              cleanupWarning:
                'Recovered the raw transcript after an interruption. Cleanup was not repeated.',
              errorMessage: undefined,
              retryCount: (retried.retryCount ?? 0) + 1,
            };
          }
        } else if (!loaded.some((row) => row.id === job.jobId || row.jobId === job.jobId)) {
          if (!job.result && !job.audioUri) continue;
          loaded.push({
            ...job.route,
            id: job.jobId,
            jobId: job.jobId,
            text: job.result?.text ?? '',
            originalText: job.result?.text,
            createdAt: now,
            updatedAt: now,
            requestContext: job.requestContext,
            status: job.result ? 'completed' : 'failed',
            audioUrl: job.result ? undefined : job.audioUri,
            cleanupWarning: job.result
              ? 'Recovered the raw transcript after an interruption. Cleanup was not repeated.'
              : undefined,
            errorMessage: job.result
              ? undefined
              : 'The provider upload was interrupted. Retry uses the original provider.',
            retryCount: 0,
          });
        }
        recoveredJobIds.push(job.jobId);
      }

      // Expire retained audio for failed rows past the retention window: drop the
      // uri so the row becomes non-retryable, and let the GC below delete the file.
      let mutated = recoveredJobIds.length > 0;
      const transcripts = loaded.map((transcript) => {
        if (
          transcript.status === 'failed' &&
          isManagedTranscriptAudioUri(transcript.audioUrl) &&
          now - transcript.updatedAt > RETAINED_AUDIO_MAX_AGE_MS
        ) {
          mutated = true;
          return { ...transcript, audioUrl: undefined };
        }
        return transcript;
      });
      if (mutated) {
        await StorageService.saveTranscripts(transcripts);
      }
      set({ transcripts, isLoading: false, isLoaded: true });
      for (const jobId of recoveredJobIds) clearKeyboardProviderRecovery(jobId);

      // Reclaim every managed file no surviving row still points at — expired
      // audio (cleared above) plus orphans from interrupted captures.
      const keepUris = transcripts
        .map((transcript) => transcript.audioUrl)
        .filter((uri): uri is string => isManagedTranscriptAudioUri(uri));
      for (const job of pendingJobs) {
        if (!recoveredJobIds.includes(job.jobId) && isManagedTranscriptAudioUri(job.audioUri)) {
          keepUris.push(job.audioUri!);
        }
      }
      const pendingJobId = activeKeyboardJobId;
      const pendingAudio =
        pendingJobId && AppGroupStorage.getItem(`keyboard_upload_audio.${pendingJobId}`);
      if (
        pendingAudio &&
        isManagedTranscriptAudioUri(pendingAudio) &&
        !keepUris.includes(pendingAudio)
      )
        keepUris.push(pendingAudio);
      await garbageCollectTranscriptAudio(keepUris).catch((error) => {
        if (__DEV__) {
          console.warn('[transcripts] audio garbage collection failed:', error);
        }
      });
    } catch (error) {
      set({ error: (error as Error).message, isLoading: false });
    }
  };

  return {
    transcripts: [],
    currentTranscript: null,
    isLoading: false,
    isLoaded: false,
    error: null,

    loadTranscripts: () => {
      inFlightLoad ??= exclusive(readHistory).finally(() => {
        inFlightLoad = null;
      });
      return inFlightLoad;
    },

    addTranscript: async (transcript) => {
      await ensureLoaded();
      const id = transcript.id ?? createTranscriptId();
      const now = Date.now();
      const newTranscript: Transcript = {
        ...transcript,
        id,
        // A completed transcript is never retried, so it has no use for its audio.
        audioUrl: undefined,
        status: 'completed',
        errorMessage: undefined,
        retryCount: transcript.retryCount ?? 0,
        createdAt: now,
        updatedAt: now,
      };

      set({ isLoading: true, error: null });
      try {
        await exclusive(async () => {
          const transcripts = [
            ...get().transcripts.filter((existing) => existing.id !== newTranscript.id),
            newTranscript,
          ];
          await StorageService.saveTranscripts(transcripts);
          set({ transcripts, isLoading: false });
        });
        if (newTranscript.jobId) clearKeyboardProviderRecovery(newTranscript.jobId);
        if (newTranscript.text.trim()) {
          logTranscriptionCompleted({
            source: newTranscript.requestContext ?? 'recording',
            provider: newTranscript.provider,
          });
        }
        // Release any retained copy (e.g. a previously-failed row with the same id
        // whose retry just succeeded) once the completed state is persisted.
        // Best-effort: deletion is idempotent and a leftover file is only waste.
        await deleteManagedTranscriptAudio(transcript.audioUrl).catch((cleanupError) => {
          if (__DEV__) {
            console.warn('[transcripts] failed to release retained audio:', cleanupError);
          }
        });
      } catch (error) {
        set({ error: (error as Error).message, isLoading: false });
        throw error;
      }
    },

    addFailedTranscript: async (transcript) => {
      await ensureLoaded();
      return exclusive(async () => {
        const id = transcript.id ?? createTranscriptId();
        const now = Date.now();
        const audioUrl = await ensureRetainedAudio(
          id,
          transcript.audioUrl,
          transcript.audioFileName,
          transcript.audioMimeType,
        );
        if (!audioUrl) {
          throw new Error('Audio file not found');
        }

        const failedTranscript: Transcript = {
          id,
          text: '',
          createdAt: now,
          updatedAt: now,
          audioUrl,
          audioFileName: transcript.audioFileName,
          audioMimeType: transcript.audioMimeType,
          duration: transcript.duration,
          provider: transcript.provider,
          inferenceRoute: transcript.inferenceRoute,
          cleanupRoute: transcript.cleanupRoute,
          cleanupUnavailable: transcript.cleanupUnavailable,
          status: 'failed',
          errorMessage: transcript.errorMessage,
          requestContext: transcript.requestContext,
          keyboardTone: transcript.keyboardTone,
          jobId: transcript.jobId,
          retryCount: 0,
        };

        set({ isLoading: true, error: null });
        try {
          const transcripts = [
            ...get().transcripts.filter((existing) => existing.id !== id),
            failedTranscript,
          ];
          await StorageService.saveTranscripts(transcripts);
          set({ transcripts, isLoading: false });
          if (failedTranscript.jobId) clearKeyboardProviderRecovery(failedTranscript.jobId);
        } catch (error) {
          set({ error: (error as Error).message, isLoading: false });
          throw error;
        }
      });
    },

    retryTranscript: async (id) => {
      await ensureLoaded();
      const current = get().transcripts.find((t) => t.id === id);
      if (!current) throw new Error('Transcript not found');
      if (!current.audioUrl) throw new Error('Audio file not found');

      if (!(await transcriptAudioExists(current.audioUrl))) {
        const message = 'Audio file not found';
        await get().updateTranscript(id, {
          status: 'failed',
          errorMessage: message,
          retryCount: (current.retryCount ?? 0) + 1,
        });
        throw new Error(message);
      }

      if (current.provider === 'byok' && !current.inferenceRoute) {
        throw new Error('The original provider route is unavailable. Start a new transcription.');
      }
      const { activeMode } = useProcessingModeStore.getState();
      const scope = current.requestContext === 'file' ? 'upload' : 'dictation';
      // Provider recordings keep their original route. Cloud and On-Device
      // recordings follow the current mode, including the provider it now selects;
      // in Local Mode, uploads follow what is chosen for them.
      const rerouted =
        current.provider !== 'byok' &&
        (activeMode === 'providers' || (activeMode === 'private' && scope === 'upload'))
          ? snapshotTranscriptionJob(scope)
          : undefined;
      const provider =
        rerouted?.provider ??
        (current.provider === 'byok' ? 'byok' : activeMode === 'private' ? 'local' : 'cloud');
      const inferenceRoute = rerouted ? rerouted.inferenceRoute : current.inferenceRoute;
      const textRoutes =
        rerouted ?? (provider === current.provider ? current : snapshotTextInference(provider));
      if (provider !== 'local' && activeMode === 'private' && scope === 'dictation') {
        throw new Error(
          'This recording used a remote provider. Change Dictation from On-Device in Account → AI Models to retry its original route.',
        );
      }
      const retryCount = (current.retryCount ?? 0) + 1;
      const retryJobId = `${current.jobId ?? id}-retry-${Date.now()}`;
      const tempUris: string[] = [];
      let audioUri = current.audioUrl;
      let fileName = current.audioFileName;
      let mimeType = current.audioMimeType;

      set({ isLoading: true, error: null });
      try {
        if (provider === 'local' && !isWavAudioFile(current) && AudioTools.isAvailable()) {
          const transcoded = await AudioTools.transcodeToWav(current.audioUrl);
          audioUri = transcoded.uri;
          fileName = `${id}-retry.wav`;
          mimeType = 'audio/wav';
          tempUris.push(transcoded.uri);
        }

        const processed = await transcribeAndCleanup({
          audioUri,
          provider,
          inferenceRoute,
          cleanupRoute:
            textRoutes.cleanupRoute ??
            (provider !== 'byok'
              ? { mode: provider === 'local' ? 'local' : 'openwhispr', scope: 'cleanup' }
              : undefined),
          cleanupUnavailable:
            textRoutes.cleanupUnavailable ??
            (provider === 'byok' && !textRoutes.cleanupRoute
              ? 'The original cleanup route is unavailable. Your raw transcript is saved.'
              : undefined),
          language: getPreferredTranscriptionLanguage(),
          fileName,
          mimeType,
          jobId: retryJobId,
          requestContext: current.requestContext ?? 'recording',
          keyboardTone: current.requestContext === 'keyboard' ? current.keyboardTone : undefined,
        });

        const updated: Transcript = {
          ...current,
          text: processed.text,
          originalText: processed.originalText,
          duration: processed.transcription.duration,
          provider: processed.transcription.provider,
          inferenceRoute: processed.transcription.inferenceRoute,
          cleanupRoute: processed.transcription.cleanupRoute,
          cleanupUnavailable: processed.transcription.cleanupUnavailable,
          cleanupWarning: processed.transcription.cleanupWarning,
          // Retry succeeded: the row is completed and no longer retryable.
          audioUrl: undefined,
          status: 'completed',
          errorMessage: undefined,
          retryCount,
          updatedAt: Date.now(),
        };
        await exclusive(async () => {
          const transcripts = get().transcripts.map((t) => (t.id === id ? updated : t));
          await StorageService.saveTranscripts(transcripts);
          set({ transcripts, isLoading: false });
        });
        if (updated.text.trim()) {
          logTranscriptionCompleted({
            source: updated.requestContext ?? 'recording',
            provider: updated.provider,
          });
        }
        // Release the retained audio once the completed state is persisted, the
        // same way a first-pass success does.
        await deleteManagedTranscriptAudio(current.audioUrl).catch((cleanupError) => {
          if (__DEV__) {
            console.warn('[transcripts] failed to release retained audio:', cleanupError);
          }
        });
        return updated;
      } catch (error) {
        const updated: Transcript = {
          ...current,
          // A rerouted Cloud or On-Device recording keeps its own route, so the next
          // retry follows the current mode instead of staying on this provider.
          ...(rerouted
            ? {}
            : {
                provider,
                inferenceRoute,
                cleanupRoute: textRoutes.cleanupRoute,
                cleanupUnavailable: textRoutes.cleanupUnavailable,
              }),
          status: 'failed',
          errorMessage: toFriendlyTranscriptionErrorMessage(error),
          retryCount,
          updatedAt: Date.now(),
        };
        await exclusive(async () => {
          const transcripts = get().transcripts.map((t) => (t.id === id ? updated : t));
          await StorageService.saveTranscripts(transcripts);
          set({ transcripts, isLoading: false });
        });
        throw error;
      } finally {
        // The row keeps its own audio and result, so the retry's recovery entry is
        // never needed once the retry settles.
        clearKeyboardProviderRecovery(retryJobId);
        await AudioTools.cleanup(tempUris).catch((cleanupError) => {
          if (__DEV__) {
            console.warn('[transcripts] failed to clean up retry temp audio:', cleanupError);
          }
        });
      }
    },

    updateTranscript: async (id, updates) => {
      set({ isLoading: true, error: null });
      try {
        await ensureLoaded();
        await exclusive(async () => {
          const transcripts = get().transcripts.map((t) =>
            t.id === id ? { ...t, ...updates, updatedAt: Date.now() } : t,
          );
          await StorageService.saveTranscripts(transcripts);
          set({ transcripts, isLoading: false });
        });
      } catch (error) {
        set({ error: (error as Error).message, isLoading: false });
      }
    },

    deleteTranscript: async (id) => {
      set({ isLoading: true, error: null });
      try {
        await ensureLoaded();
        await exclusive(async () => {
          const target = get().transcripts.find((t) => t.id === id);
          const transcripts = get().transcripts.filter((t) => t.id !== id);
          await StorageService.saveTranscripts(transcripts);
          await deleteManagedTranscriptAudio(target?.audioUrl);
          set({ transcripts, isLoading: false });
        });
      } catch (error) {
        set({ error: (error as Error).message, isLoading: false });
      }
    },

    setCurrentTranscript: (transcript) => {
      set({ currentTranscript: transcript });
    },

    clearTranscripts: async () => {
      set({ isLoading: true, error: null });
      try {
        await ensureLoaded();
        await exclusive(async () => {
          const current = get().transcripts;
          await StorageService.clearTranscripts();
          await Promise.all(current.map((t) => deleteManagedTranscriptAudio(t.audioUrl)));
          set({ transcripts: [], isLoading: false });
        });
      } catch (error) {
        set({ error: (error as Error).message, isLoading: false });
      }
    },
  };
});