Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(recording): persist multi-language transcription selection (#4321)

Ansh Groverf669ae0de0e5
diff --git a/apps/screenpipe-app-tauri/components/settings/recording-settings.tsx b/apps/screenpipe-app-tauri/components/settings/recording-settings.tsx
index e734282d5..c731a367e 100644
--- a/apps/screenpipe-app-tauri/components/settings/recording-settings.tsx
+++ b/apps/screenpipe-app-tauri/components/settings/recording-settings.tsx
@@ -1927,6 +1927,15 @@ export function RecordingSettings() {
: languageSupportIsLimited
? `Restricts transcription to selected languages supported by ${languageSupportLabel}`
: "Restricts transcription to selected";
+ const selectedLanguageNames = settings.languages
+ .map((code) => supportedLanguageOptions.find((language) => language.code === code)?.name ?? code)
+ .join(", ");
+ const languageTriggerLabel =
+ settings.languages.length === 0
+ ? "Auto-detect"
+ : settings.languages.length <= 2
+ ? selectedLanguageNames
+ : `${settings.languages.length} selected`;
// Add new state to track if settings have changed
const [hasUnsavedChanges, setHasUnsavedChanges] = useState(false);
@@ -2002,14 +2011,16 @@ export function RecordingSettings() {
useEffect(() => {
const previousSnapshot = languageSelectionSnapshotRef.current;
- if (previousSnapshot.supportKey !== languageSupportKey) {
+ const supportKeyChanged = previousSnapshot.supportKey !== languageSupportKey;
+ if (supportKeyChanged) {
languageSelectionsBySupportKeyRef.current[previousSnapshot.supportKey] = [
...previousSnapshot.languages,
];
}
- const preferredLanguages =
- languageSelectionsBySupportKeyRef.current[languageSupportKey];
+ const preferredLanguages = supportKeyChanged
+ ? languageSelectionsBySupportKeyRef.current[languageSupportKey]
+ : undefined;
const resolvedLanguages = resolveLanguageSelectionForTranscriptionEngine(
settings.languages,
languageSupportEngine,
@@ -2351,7 +2362,12 @@ export function RecordingSettings() {
};
- const handleLanguageChange = (currentValue: Language) => {
+ const handleLanguageChange = (currentValue: Language | null) => {
+ if (!currentValue) {
+ handleSettingsChange({ languages: [] });
+ return;
+ }
+
const updatedLanguages = settings.languages.includes(currentValue)
? settings.languages.filter((id) => id !== currentValue)
: [...settings.languages, currentValue];
@@ -3339,7 +3355,7 @@ Your screen is a pipe. Everything you see, hear, and type flows through it. Scre
<Popover open={openLanguages} onOpenChange={setOpenLanguages}>
<PopoverTrigger asChild>
<Button variant="outline" size="sm" className="h-7 text-xs">
- {settings.languages.length > 0 ? `${settings.languages.length} selected` : "Auto-detect"}
+ {languageTriggerLabel}
<ChevronsUpDown className="ml-1 h-3 w-3 opacity-50" />
</Button>
</PopoverTrigger>
@@ -3349,6 +3365,10 @@ Your screen is a pipe. Everything you see, hear, and type flows through it. Scre
<CommandList>
<CommandEmpty>No languages found.</CommandEmpty>
<CommandGroup>
+ <CommandItem value="auto-detect" onSelect={() => handleLanguageChange(null)}>
+ <Check className={cn("mr-2 h-3 w-3", settings.languages.length === 0 ? "opacity-100" : "opacity-0")} />
+ <span className="text-xs">Auto-detect</span>
+ </CommandItem>
{supportedLanguageOptions.map((language) => (
<CommandItem key={language.code} value={language.code} onSelect={() => handleLanguageChange(language.code)}>
<Check className={cn("mr-2 h-3 w-3", settings.languages.includes(language.code) ? "opacity-100" : "opacity-0")} />
diff --git a/packages/ai-gateway/src/handlers/transcription.ts b/packages/ai-gateway/src/handlers/transcription.ts
index 4db1b0ea5..5b66695f5 100644
--- a/packages/ai-gateway/src/handlers/transcription.ts
+++ b/packages/ai-gateway/src/handlers/transcription.ts
@@ -48,6 +48,51 @@ export async function handleABTestAdmin(request: Request, env: Env): Promise<Res
return createSuccessResponse(summary);
}
+function parseLanguageValues(values: Array<string | null>): string[] {
+ const seen = new Set<string>();
+ const languages: string[] = [];
+
+ for (const value of values) {
+ for (const language of (value || '').split(',')) {
+ const normalized = language.trim();
+ const lower = normalized.toLowerCase();
+ if (!normalized || lower === 'true' || lower === 'false' || lower === 'auto' || lower === 'auto-detect') {
+ continue;
+ }
+ if (!seen.has(normalized)) {
+ seen.add(normalized);
+ languages.push(normalized);
+ }
+ }
+ }
+
+ return languages;
+}
+
+function getTranscriptionLanguages(request: Request, fallback: string[] = []): string[] {
+ const url = new URL(request.url);
+ const queryLanguages = parseLanguageValues([
+ ...url.searchParams.getAll('language'),
+ ...url.searchParams.getAll('languages'),
+ ...url.searchParams.getAll('detect_language'),
+ ]);
+ if (queryLanguages.length > 0) {
+ return queryLanguages;
+ }
+
+ const headerLanguages = parseLanguageValues([
+ request.headers.get('language'),
+ request.headers.get('languages'),
+ request.headers.get('detect_language'),
+ ]);
+ return headerLanguages.length > 0 ? headerLanguages : fallback;
+}
+
+function getSampleRate(request: Request, fallback = '16000'): string {
+ const url = new URL(request.url);
+ return url.searchParams.get('sample_rate') || request.headers.get('sample_rate') || fallback;
+}
+
/**
* Handles transcription with A/B test routing between Deepgram and self-hosted Whisper.
* Falls back to Deepgram if Whisper fails.
@@ -60,8 +105,8 @@ async function handleDeepgramTranscription(
): Promise<Response> {
try {
const audioBuffer = await request.arrayBuffer();
- const languages = request.headers.get('detect_language')?.split(',') || [];
- const sampleRate = request.headers.get('sample_rate') || '16000';
+ const languages = getTranscriptionLanguages(request);
+ const sampleRate = getSampleRate(request);
const contentType = request.headers.get('Content-Type') || 'audio/wav';
const abReq: TranscriptionRequest = { audioBuffer, contentType, sampleRate, languages };
@@ -89,8 +134,8 @@ async function handleDeepgramTranscription(
async function handleGoogleTranscription(request: Request, env: Env): Promise<Response> {
try {
const audioBuffer = await request.arrayBuffer();
- const languages = request.headers.get('detect_language')?.split(',') || ['en-US'];
- const sampleRate = parseInt(request.headers.get('sample_rate') || '16000', 10);
+ const languages = getTranscriptionLanguages(request, ['en-US']);
+ const sampleRate = parseInt(getSampleRate(request), 10);
const vertexProvider = new VertexAIProvider(
env.VERTEX_SERVICE_ACCOUNT_JSON,
@@ -192,7 +237,7 @@ async function handleGoogleTranscription(request: Request, env: Env): Promise<Re
async function handleChirp2Transcription(request: Request, env: Env): Promise<Response> {
try {
const audioBuffer = await request.arrayBuffer();
- const languages = request.headers.get('detect_language')?.split(',') || ['en-US'];
+ const languages = getTranscriptionLanguages(request, ['en-US']);
const vertexProvider = new VertexAIProvider(
env.VERTEX_SERVICE_ACCOUNT_JSON,
diff --git a/packages/ai-gateway/src/test/transcription-ab.unit.test.ts b/packages/ai-gateway/src/test/transcription-ab.unit.test.ts
index 0e31e910f..629f97ac1 100644
--- a/packages/ai-gateway/src/test/transcription-ab.unit.test.ts
+++ b/packages/ai-gateway/src/test/transcription-ab.unit.test.ts
@@ -21,6 +21,7 @@ import {
extractTranscript,
runTranscriptionABTest,
} from '../services/transcription-ab';
+import { handleFileTranscription } from '../handlers/transcription';
// ─── Config parsing ─────────────────────────────────────────────────────────
@@ -209,6 +210,116 @@ describe('callDeepgram', () => {
});
});
+describe('handleFileTranscription', () => {
+ it('reads repeated detect_language query params from desktop clients', async () => {
+ const originalFetch = globalThis.fetch;
+ const urls: string[] = [];
+
+ globalThis.fetch = (async (input: RequestInfo | URL) => {
+ urls.push(String(input));
+ return new Response(
+ JSON.stringify({
+ results: {
+ channels: [{
+ alternatives: [{ transcript: 'hello world', confidence: 0.9 }],
+ }],
+ },
+ }),
+ { status: 200, headers: { 'Content-Type': 'application/json' } },
+ );
+ }) as typeof fetch;
+
+ try {
+ const response = await handleFileTranscription(
+ new Request('https://ai.example/v1/listen?detect_language=en&detect_language=hi&sample_rate=16000', {
+ method: 'POST',
+ body: new Uint8Array([1, 2, 3]),
+ headers: { 'Content-Type': 'audio/mpeg' },
+ }),
+ { DEEPGRAM_API_KEY: 'dg-test-key' } as any,
+ );
+
+ expect(response.status).toBe(200);
+ const url = new URL(urls[0]);
+ expect(url.searchParams.getAll('detect_language')).toEqual(['en', 'hi']);
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+ });
+
+ it('reads a single language query param from desktop clients', async () => {
+ const originalFetch = globalThis.fetch;
+ const urls: string[] = [];
+
+ globalThis.fetch = (async (input: RequestInfo | URL) => {
+ urls.push(String(input));
+ return new Response(
+ JSON.stringify({
+ results: {
+ channels: [{
+ alternatives: [{ transcript: 'hello world', confidence: 0.9 }],
+ }],
+ },
+ }),
+ { status: 200, headers: { 'Content-Type': 'application/json' } },
+ );
+ }) as typeof fetch;
+
+ try {
+ const response = await handleFileTranscription(
+ new Request('https://ai.example/v1/listen?language=en&sample_rate=16000', {
+ method: 'POST',
+ body: new Uint8Array([1, 2, 3]),
+ headers: { 'Content-Type': 'audio/mpeg' },
+ }),
+ { DEEPGRAM_API_KEY: 'dg-test-key' } as any,
+ );
+
+ expect(response.status).toBe(200);
+ const url = new URL(urls[0]);
+ expect(url.searchParams.getAll('detect_language')).toEqual(['en']);
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+ });
+
+ it('treats detect_language=true as auto-detect', async () => {
+ const originalFetch = globalThis.fetch;
+ const urls: string[] = [];
+
+ globalThis.fetch = (async (input: RequestInfo | URL) => {
+ urls.push(String(input));
+ return new Response(
+ JSON.stringify({
+ results: {
+ channels: [{
+ alternatives: [{ transcript: 'hello world', confidence: 0.9 }],
+ }],
+ },
+ }),
+ { status: 200, headers: { 'Content-Type': 'application/json' } },
+ );
+ }) as typeof fetch;
+
+ try {
+ const response = await handleFileTranscription(
+ new Request('https://ai.example/v1/listen?detect_language=true&sample_rate=16000', {
+ method: 'POST',
+ body: new Uint8Array([1, 2, 3]),
+ headers: { 'Content-Type': 'audio/mpeg' },
+ }),
+ { DEEPGRAM_API_KEY: 'dg-test-key' } as any,
+ );
+
+ expect(response.status).toBe(200);
+ const url = new URL(urls[0]);
+ expect(url.searchParams.getAll('detect_language')).toEqual([]);
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+ });
+});
+
describe('runTranscriptionABTest', () => {
it('falls back to configured self-hosted transcription when Deepgram fails', async () => {
const originalFetch = globalThis.fetch;