YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(recording): persist multi-language transcription selection (#4321)
Ansh Groverf669ae0de0e5
diff --git a/apps/screenpipe-app-tauri/components/settings/recording-settings.tsx b/apps/screenpipe-app-tauri/components/settings/recording-settings.tsxindex e734282d5..c731a367e 100644--- a/apps/screenpipe-app-tauri/components/settings/recording-settings.tsx+++ b/apps/screenpipe-app-tauri/components/settings/recording-settings.tsx@@ -1927,6 +1927,15 @@ export function RecordingSettings() {: languageSupportIsLimited? `Restricts transcription to selected languages supported by ${languageSupportLabel}`: "Restricts transcription to selected";+ const selectedLanguageNames = settings.languages+ .map((code) => supportedLanguageOptions.find((language) => language.code === code)?.name ?? code)+ .join(", ");+ const languageTriggerLabel =+ settings.languages.length === 0+ ? "Auto-detect"+ : settings.languages.length <= 2+ ? selectedLanguageNames+ : `${settings.languages.length} selected`;// Add new state to track if settings have changedconst [hasUnsavedChanges, setHasUnsavedChanges] = useState(false);@@ -2002,14 +2011,16 @@ export function RecordingSettings() {useEffect(() => {const previousSnapshot = languageSelectionSnapshotRef.current;- if (previousSnapshot.supportKey !== languageSupportKey) {+ const supportKeyChanged = previousSnapshot.supportKey !== languageSupportKey;+ if (supportKeyChanged) {languageSelectionsBySupportKeyRef.current[previousSnapshot.supportKey] = [...previousSnapshot.languages,];}- const preferredLanguages =- languageSelectionsBySupportKeyRef.current[languageSupportKey];+ const preferredLanguages = supportKeyChanged+ ? languageSelectionsBySupportKeyRef.current[languageSupportKey]+ : undefined;const resolvedLanguages = resolveLanguageSelectionForTranscriptionEngine(settings.languages,languageSupportEngine,@@ -2351,7 +2362,12 @@ export function RecordingSettings() {};- const handleLanguageChange = (currentValue: Language) => {+ const handleLanguageChange = (currentValue: Language | null) => {+ if (!currentValue) {+ handleSettingsChange({ languages: [] });+ return;+ }+const updatedLanguages = settings.languages.includes(currentValue)? settings.languages.filter((id) => id !== currentValue): [...settings.languages, currentValue];@@ -3339,7 +3355,7 @@ Your screen is a pipe. Everything you see, hear, and type flows through it. Scre<Popover open={openLanguages} onOpenChange={setOpenLanguages}><PopoverTrigger asChild><Button variant="outline" size="sm" className="h-7 text-xs">- {settings.languages.length > 0 ? `${settings.languages.length} selected` : "Auto-detect"}+ {languageTriggerLabel}<ChevronsUpDown className="ml-1 h-3 w-3 opacity-50" /></Button></PopoverTrigger>@@ -3349,6 +3365,10 @@ Your screen is a pipe. Everything you see, hear, and type flows through it. Scre<CommandList><CommandEmpty>No languages found.</CommandEmpty><CommandGroup>+ <CommandItem value="auto-detect" onSelect={() => handleLanguageChange(null)}>+ <Check className={cn("mr-2 h-3 w-3", settings.languages.length === 0 ? "opacity-100" : "opacity-0")} />+ <span className="text-xs">Auto-detect</span>+ </CommandItem>{supportedLanguageOptions.map((language) => (<CommandItem key={language.code} value={language.code} onSelect={() => handleLanguageChange(language.code)}><Check className={cn("mr-2 h-3 w-3", settings.languages.includes(language.code) ? "opacity-100" : "opacity-0")} />diff --git a/packages/ai-gateway/src/handlers/transcription.ts b/packages/ai-gateway/src/handlers/transcription.tsindex 4db1b0ea5..5b66695f5 100644--- a/packages/ai-gateway/src/handlers/transcription.ts+++ b/packages/ai-gateway/src/handlers/transcription.ts@@ -48,6 +48,51 @@ export async function handleABTestAdmin(request: Request, env: Env): Promise<Resreturn createSuccessResponse(summary);}+function parseLanguageValues(values: Array<string | null>): string[] {+ const seen = new Set<string>();+ const languages: string[] = [];++ for (const value of values) {+ for (const language of (value || '').split(',')) {+ const normalized = language.trim();+ const lower = normalized.toLowerCase();+ if (!normalized || lower === 'true' || lower === 'false' || lower === 'auto' || lower === 'auto-detect') {+ continue;+ }+ if (!seen.has(normalized)) {+ seen.add(normalized);+ languages.push(normalized);+ }+ }+ }++ return languages;+}++function getTranscriptionLanguages(request: Request, fallback: string[] = []): string[] {+ const url = new URL(request.url);+ const queryLanguages = parseLanguageValues([+ ...url.searchParams.getAll('language'),+ ...url.searchParams.getAll('languages'),+ ...url.searchParams.getAll('detect_language'),+ ]);+ if (queryLanguages.length > 0) {+ return queryLanguages;+ }++ const headerLanguages = parseLanguageValues([+ request.headers.get('language'),+ request.headers.get('languages'),+ request.headers.get('detect_language'),+ ]);+ return headerLanguages.length > 0 ? headerLanguages : fallback;+}++function getSampleRate(request: Request, fallback = '16000'): string {+ const url = new URL(request.url);+ return url.searchParams.get('sample_rate') || request.headers.get('sample_rate') || fallback;+}+/*** Handles transcription with A/B test routing between Deepgram and self-hosted Whisper.* Falls back to Deepgram if Whisper fails.@@ -60,8 +105,8 @@ async function handleDeepgramTranscription(): Promise<Response> {try {const audioBuffer = await request.arrayBuffer();- const languages = request.headers.get('detect_language')?.split(',') || [];- const sampleRate = request.headers.get('sample_rate') || '16000';+ const languages = getTranscriptionLanguages(request);+ const sampleRate = getSampleRate(request);const contentType = request.headers.get('Content-Type') || 'audio/wav';const abReq: TranscriptionRequest = { audioBuffer, contentType, sampleRate, languages };@@ -89,8 +134,8 @@ async function handleDeepgramTranscription(async function handleGoogleTranscription(request: Request, env: Env): Promise<Response> {try {const audioBuffer = await request.arrayBuffer();- const languages = request.headers.get('detect_language')?.split(',') || ['en-US'];- const sampleRate = parseInt(request.headers.get('sample_rate') || '16000', 10);+ const languages = getTranscriptionLanguages(request, ['en-US']);+ const sampleRate = parseInt(getSampleRate(request), 10);const vertexProvider = new VertexAIProvider(env.VERTEX_SERVICE_ACCOUNT_JSON,@@ -192,7 +237,7 @@ async function handleGoogleTranscription(request: Request, env: Env): Promise<Reasync function handleChirp2Transcription(request: Request, env: Env): Promise<Response> {try {const audioBuffer = await request.arrayBuffer();- const languages = request.headers.get('detect_language')?.split(',') || ['en-US'];+ const languages = getTranscriptionLanguages(request, ['en-US']);const vertexProvider = new VertexAIProvider(env.VERTEX_SERVICE_ACCOUNT_JSON,diff --git a/packages/ai-gateway/src/test/transcription-ab.unit.test.ts b/packages/ai-gateway/src/test/transcription-ab.unit.test.tsindex 0e31e910f..629f97ac1 100644--- a/packages/ai-gateway/src/test/transcription-ab.unit.test.ts+++ b/packages/ai-gateway/src/test/transcription-ab.unit.test.ts@@ -21,6 +21,7 @@ import {extractTranscript,runTranscriptionABTest,} from '../services/transcription-ab';+import { handleFileTranscription } from '../handlers/transcription';// ─── Config parsing ─────────────────────────────────────────────────────────@@ -209,6 +210,116 @@ describe('callDeepgram', () => {});});+describe('handleFileTranscription', () => {+ it('reads repeated detect_language query params from desktop clients', async () => {+ const originalFetch = globalThis.fetch;+ const urls: string[] = [];++ globalThis.fetch = (async (input: RequestInfo | URL) => {+ urls.push(String(input));+ return new Response(+ JSON.stringify({+ results: {+ channels: [{+ alternatives: [{ transcript: 'hello world', confidence: 0.9 }],+ }],+ },+ }),+ { status: 200, headers: { 'Content-Type': 'application/json' } },+ );+ }) as typeof fetch;++ try {+ const response = await handleFileTranscription(+ new Request('https://ai.example/v1/listen?detect_language=en&detect_language=hi&sample_rate=16000', {+ method: 'POST',+ body: new Uint8Array([1, 2, 3]),+ headers: { 'Content-Type': 'audio/mpeg' },+ }),+ { DEEPGRAM_API_KEY: 'dg-test-key' } as any,+ );++ expect(response.status).toBe(200);+ const url = new URL(urls[0]);+ expect(url.searchParams.getAll('detect_language')).toEqual(['en', 'hi']);+ } finally {+ globalThis.fetch = originalFetch;+ }+ });++ it('reads a single language query param from desktop clients', async () => {+ const originalFetch = globalThis.fetch;+ const urls: string[] = [];++ globalThis.fetch = (async (input: RequestInfo | URL) => {+ urls.push(String(input));+ return new Response(+ JSON.stringify({+ results: {+ channels: [{+ alternatives: [{ transcript: 'hello world', confidence: 0.9 }],+ }],+ },+ }),+ { status: 200, headers: { 'Content-Type': 'application/json' } },+ );+ }) as typeof fetch;++ try {+ const response = await handleFileTranscription(+ new Request('https://ai.example/v1/listen?language=en&sample_rate=16000', {+ method: 'POST',+ body: new Uint8Array([1, 2, 3]),+ headers: { 'Content-Type': 'audio/mpeg' },+ }),+ { DEEPGRAM_API_KEY: 'dg-test-key' } as any,+ );++ expect(response.status).toBe(200);+ const url = new URL(urls[0]);+ expect(url.searchParams.getAll('detect_language')).toEqual(['en']);+ } finally {+ globalThis.fetch = originalFetch;+ }+ });++ it('treats detect_language=true as auto-detect', async () => {+ const originalFetch = globalThis.fetch;+ const urls: string[] = [];++ globalThis.fetch = (async (input: RequestInfo | URL) => {+ urls.push(String(input));+ return new Response(+ JSON.stringify({+ results: {+ channels: [{+ alternatives: [{ transcript: 'hello world', confidence: 0.9 }],+ }],+ },+ }),+ { status: 200, headers: { 'Content-Type': 'application/json' } },+ );+ }) as typeof fetch;++ try {+ const response = await handleFileTranscription(+ new Request('https://ai.example/v1/listen?detect_language=true&sample_rate=16000', {+ method: 'POST',+ body: new Uint8Array([1, 2, 3]),+ headers: { 'Content-Type': 'audio/mpeg' },+ }),+ { DEEPGRAM_API_KEY: 'dg-test-key' } as any,+ );++ expect(response.status).toBe(200);+ const url = new URL(urls[0]);+ expect(url.searchParams.getAll('detect_language')).toEqual([]);+ } finally {+ globalThis.fetch = originalFetch;+ }+ });+});+describe('runTranscriptionABTest', () => {it('falls back to configured self-hosted transcription when Deepgram fails', async () => {const originalFetch = globalThis.fetch;