Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

11.0 KB
Raw Blame
import { withSessionRefresh } from "../lib/auth";
import {
  resolveTranscriptionRoute,
  type ManagedTranscriptionResolution,
  type TranscriptionRoute,
} from "../helpers/transcriptionRoute";
import {
  getManagedTranscriptionResolution,
  isManagedTranscriptionActive,
} from "./managedTranscription";
import { getTranscriptionProviders } from "../models/ModelRegistry";
import type { LocalTranscriptionProvider } from "../types/electron";

export interface FileTranscriptionResult {
  success: boolean;
  text?: string;
  error?: string;
  code?: string;
  messageKey?: string;
  diarized?: boolean;
  // The transcript succeeded, but requested speaker labels could not be applied.
  diarizationWarning?: boolean;
  warning?: string;
  // Set alongside `warning` by the chunked cloud path: how much audio was lost.
  failedChunks?: number;
  totalChunks?: number;
  // Measured duration of the source audio, for persisting as
  // audio_duration_seconds. Only transcribeFileWithSpeakers sets it.
  durationSeconds?: number | null;
  // Segment-level timing from BYOK providers that support it (opts.timestamps
  // or BYOK diarization), with local diarization's speakers when that ran, or
  // the merged speaker turns when the provider returned text only.
  segments?: Array<{ text: string; start: number; end: number; speaker?: string }>;
}

export interface DiarizationSettings {
  enabled: boolean;
  // Local sherpa-onnx models present; BYOK-native diarization doesn't need them.
  localModelsReady: boolean;
  numSpeakers: number | null;
}

export interface FileTranscriptionConfig {
  useLocalWhisper: boolean;
  localTranscriptionProvider: string;
  whisperModel: string;
  parakeetModel: string;
  cohereModel: string;
  isOpenWhisprCloud: boolean;
  getApiKey: () => string;
  cloudTranscriptionProvider: string;
  cloudTranscriptionBaseUrl: string;
  cloudTranscriptionModel: string;
  language: string;
  cortiEnvironment?: string;
  cortiTenant?: string;
  transcriptionMode?: string;
  remoteTranscriptionUrl?: string;
  remoteTranscriptionModel?: string;
}

export interface TranscriptionApiKeys {
  openaiApiKey: string;
  groqApiKey: string;
  xaiApiKey: string;
  mistralApiKey: string;
  geminiApiKey: string;
  tinfoilApiKey: string;
  deepgramApiKey: string;
  assemblyaiApiKey: string;
  customTranscriptionApiKey?: string;
}

export function getTranscriptionApiKey(provider: string, keys: TranscriptionApiKeys): string {
  switch (provider) {
    case "openai":
      return keys.openaiApiKey;
    case "groq":
      return keys.groqApiKey;
    case "xai":
      return keys.xaiApiKey;
    case "mistral":
      return keys.mistralApiKey;
    case "gemini":
      return keys.geminiApiKey;
    case "tinfoil":
      return keys.tinfoilApiKey;
    case "deepgram":
      return keys.deepgramApiKey;
    case "assemblyai":
      return keys.assemblyaiApiKey;
    case "custom":
      return keys.customTranscriptionApiKey || "";
    default:
      return "";
  }
}

// Pre-flight through the shared resolver: code-carrying errors (incl. the
// Tinfoil-URL and fail-closed custom guards) surface here without an IPC
// round-trip; the main-process handler re-resolves the same fields as
// defense in depth.
export function resolveFileTranscriptionRoute(
  cfg: FileTranscriptionConfig,
  managed: ManagedTranscriptionResolution | null = null
): TranscriptionRoute {
  return resolveTranscriptionRoute({
    settings: {
      transcriptionMode: cfg.transcriptionMode,
      remoteTranscriptionUrl: cfg.remoteTranscriptionUrl,
      remoteTranscriptionModel: cfg.remoteTranscriptionModel,
      cloudTranscriptionProvider: cfg.cloudTranscriptionProvider,
      cloudTranscriptionModel: cfg.cloudTranscriptionModel,
      cloudTranscriptionBaseUrl: cfg.cloudTranscriptionBaseUrl,
      cortiEnvironment: cfg.cortiEnvironment,
      cortiTenant: cfg.cortiTenant,
    },
    providers: getTranscriptionProviders(),
    managed,
    request: { effectiveLanguage: cfg.language || undefined },
  });
}

// Single provider dispatch shared by the single-file flow and the batch queue,
// so BYOK providers receive identical options in both.
export async function transcribeFile(
  filePath: string,
  cfg: FileTranscriptionConfig,
  diarize: boolean,
  opts: { requestId?: string; timestamps?: boolean } = {}
): Promise<FileTranscriptionResult> {
  // Managed enterprise STT outranks every personal lane, OpenWhispr Cloud and
  // local included: under managed_required no audio may leave the tenant.
  const managed = getManagedTranscriptionResolution();
  if (managed?.kind === "error") {
    return {
      success: false,
      error: managed.message,
      code: managed.code,
      messageKey: managed.messageKey,
    };
  }

  if (!managed && cfg.isOpenWhisprCloud) {
    return withSessionRefresh(async () => {
      const r = await window.electronAPI.transcribeAudioFileCloud!(filePath, opts);
      if (!r.success && r.code) {
        throw Object.assign(new Error(r.error || "Cloud transcription failed"), {
          code: r.code,
        });
      }
      return r;
    });
  }

  if (!managed && cfg.useLocalWhisper) {
    const provider = cfg.localTranscriptionProvider as LocalTranscriptionProvider;
    return window.electronAPI.transcribeAudioFile(filePath, {
      provider,
      model:
        provider === "nvidia"
          ? cfg.parakeetModel
          : provider === "cohere"
            ? cfg.cohereModel
            : cfg.whisperModel,
      // Only Cohere consumes it (no auto-detect); passing it to whisper would
      // override its auto language detection.
      ...(provider === "cohere" && cfg.language ? { language: cfg.language } : {}),
      requestId: opts.requestId,
    });
  }

  const route = resolveFileTranscriptionRoute(cfg, managed);
  if (route.transport === "error") {
    return {
      success: false,
      error: route.message,
      code: route.code,
      messageKey: route.messageKey,
    };
  }

  // Self-hosted fields make the handler route to the configured server
  // (fail-closed on misconfiguration) instead of stale BYOK settings.
  return window.electronAPI.transcribeAudioFileByok!({
    filePath,
    managed: managed?.kind === "managed" ? managed : undefined,
    apiKey: cfg.getApiKey(),
    baseUrl: cfg.cloudTranscriptionBaseUrl,
    model: cfg.cloudTranscriptionModel,
    diarize: diarize || undefined,
    timestamps: opts.timestamps || undefined,
    provider: cfg.cloudTranscriptionProvider,
    language: cfg.language,
    environment: cfg.cortiEnvironment,
    tenant: cfg.cortiTenant,
    transcriptionMode: cfg.transcriptionMode,
    remoteTranscriptionUrl: cfg.remoteTranscriptionUrl,
    remoteTranscriptionModel: cfg.remoteTranscriptionModel,
  });
}

// OpenAI/Mistral BYOK handle diarization inside the transcription call itself.
// Self-hosted mode routes to the user's own server, which doesn't — those users
// get local diarization like everyone else.
export function shouldUseByokDiarize(
  cfg: FileTranscriptionConfig,
  diarizationEnabled: boolean
): boolean {
  if (isManagedTranscriptionActive()) return false;
  return (
    diarizationEnabled &&
    !cfg.useLocalWhisper &&
    !cfg.isOpenWhisprCloud &&
    cfg.transcriptionMode !== "self-hosted" &&
    (cfg.cloudTranscriptionProvider === "openai" || cfg.cloudTranscriptionProvider === "mistral")
  );
}

interface DiarizationSettingsRequest {
  enabled: boolean;
  modelsReady: boolean;
  numSpeakers: number | null;
  config: FileTranscriptionConfig;
  ensureModels: () => Promise<boolean>;
}

// Speaker detection is opt-in but its local models download lazily, so a run
// started before they land would report "couldn't be applied" without the
// diarizer ever being invoked. Fetch them first; only the routes that diarize
// server-side can skip the download.
export async function resolveDiarizationSettings({
  enabled,
  modelsReady,
  numSpeakers,
  config,
  ensureModels,
}: DiarizationSettingsRequest): Promise<DiarizationSettings> {
  const needsLocalModels = enabled && !modelsReady && !shouldUseByokDiarize(config, enabled);
  return {
    enabled,
    localModelsReady: needsLocalModels ? await ensureModels() : modelsReady,
    numSpeakers,
  };
}

type SpeakerSpan = { start: number; end: number; speaker: string };

function assignSpeakers(
  segments: NonNullable<FileTranscriptionResult["segments"]>,
  speakerSpans: SpeakerSpan[]
): NonNullable<FileTranscriptionResult["segments"]> {
  return segments.map((segment) => {
    let best = speakerSpans[0];
    let bestScore = Number.NEGATIVE_INFINITY;
    for (const span of speakerSpans) {
      // Positive: seconds of overlap. Negative: the gap to a span that doesn't overlap.
      const score = Math.min(segment.end, span.end) - Math.max(segment.start, span.start);
      if (score > bestScore) {
        best = span;
        bestScore = score;
      }
    }
    return { ...segment, speaker: best.speaker };
  });
}

// Transcribe and diarize in parallel, then merge speaker labels into the text.
// Shared by the single-file flow and the batch queue. `durationSeconds` (when the
// source knows it, e.g. URL downloads) beats inferring duration from segments.
export async function transcribeFileWithSpeakers(
  filePath: string,
  cfg: FileTranscriptionConfig,
  diarization: DiarizationSettings,
  durationSeconds?: number | null,
  opts: { requestId?: string; timestamps?: boolean } = {}
): Promise<FileTranscriptionResult> {
  const byokDiarize = shouldUseByokDiarize(cfg, diarization.enabled);
  const diarizePromise =
    diarization.enabled && diarization.localModelsReady && !byokDiarize
      ? (window.electronAPI
          .diarizeAudioFile?.(filePath, {
            numSpeakers: diarization.numSpeakers ?? undefined,
            requestId: opts.requestId,
          })
          .catch(() => null) ?? Promise.resolve(null))
      : Promise.resolve(null);

  const [transcribed, diar] = await Promise.all([
    transcribeFile(filePath, cfg, byokDiarize, opts),
    diarizePromise,
  ]);

  // The diarizer measures the converted audio, so it covers picked files whose
  // duration the caller never knew. 0/NaN mean "unknown", hence ||.
  const measuredDuration = durationSeconds || (diar?.success && diar.durationSeconds) || null;
  const result = { ...transcribed, durationSeconds: measuredDuration };

  if (!result.success || !result.text || !diarization.enabled || result.diarized) return result;
  if (!diar?.success || !diar.segments?.length) {
    return { ...result, diarizationWarning: true };
  }

  try {
    const merged = await window.electronAPI.mergeSpeakerText?.(
      diar.segments,
      result.text,
      measuredDuration || 0
    );
    if (merged?.success && merged.text) {
      // The transcript keeps who spoke when: the provider's timed segments get
      // the speaker they overlap most, and without them the merged turns
      // (placed by their share of the text) stand in.
      return {
        ...result,
        text: merged.text,
        segments: result.segments
          ? assignSpeakers(result.segments, diar.segments)
          : merged.segments,
      };
    }
  } catch {
    // Merge failure falls back to the plain transcript.
  }
  return { ...result, diarizationWarning: true };
}