A free, open source, and extensible speech-to-text application that works completely offline.
feat: implement structured outputs for post-processing providers (#706)
Chirag Agggarwal0cb8ab21624c
diff --git a/src-tauri/src/actions.rs b/src-tauri/src/actions.rsindex 53393881..f19ce58b 100644--- a/src-tauri/src/actions.rs+++ b/src-tauri/src/actions.rs@@ -12,7 +12,7 @@ use crate::utils::{};use crate::TranscriptionCoordinator;use ferrous_opencc::{config::BuiltinConfig, OpenCC};-use log::{debug, error};+use log::{debug, error, warn};use once_cell::sync::Lazy;use std::collections::HashMap;use std::sync::Arc;@@ -42,6 +42,20 @@ struct TranscribeAction {post_process: bool,}+/// Field name for structured output JSON schema+const TRANSCRIPTION_FIELD: &str = "transcription";++/// Strip invisible Unicode characters that some LLMs may insert+fn strip_invisible_chars(s: &str) -> String {+ s.replace(['\u{200B}', '\u{200C}', '\u{200D}', '\u{FEFF}'], "")+}++/// Build a system prompt from the user's prompt template.+/// Removes `${output}` placeholder since the transcription is sent as the user message.+fn build_system_prompt(prompt_template: &str) -> String {+ prompt_template.replace("${output}", "").trim().to_string()+}+async fn post_process_transcription(settings: &AppSettings, transcription: &str) -> Option<String> {let provider = match settings.active_post_process_provider().cloned() {Some(provider) => provider,@@ -98,63 +112,136 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str)provider.id, model);- // Replace ${output} variable in the prompt with the actual text- let processed_prompt = prompt.replace("${output}", transcription);- debug!("Processed prompt length: {} chars", processed_prompt.len());+ let api_key = settings+ .post_process_api_keys+ .get(&provider.id)+ .cloned()+ .unwrap_or_default();- if provider.id == APPLE_INTELLIGENCE_PROVIDER_ID {- #[cfg(all(target_os = "macos", target_arch = "aarch64"))]- {- if !apple_intelligence::check_apple_intelligence_availability() {- debug!("Apple Intelligence selected but not currently available on this device");- return None;- }+ if provider.supports_structured_output {+ debug!("Using structured outputs for provider '{}'", provider.id);++ let system_prompt = build_system_prompt(&prompt);+ let user_content = transcription.to_string();++ // Handle Apple Intelligence separately since it uses native Swift APIs+ if provider.id == APPLE_INTELLIGENCE_PROVIDER_ID {+ #[cfg(all(target_os = "macos", target_arch = "aarch64"))]+ {+ if !apple_intelligence::check_apple_intelligence_availability() {+ debug!(+ "Apple Intelligence selected but not currently available on this device"+ );+ return None;+ }- let token_limit = model.trim().parse::<i32>().unwrap_or(0);- return match apple_intelligence::process_text(&processed_prompt, token_limit) {- Ok(result) => {- if result.trim().is_empty() {- debug!("Apple Intelligence returned an empty response");+ let token_limit = model.trim().parse::<i32>().unwrap_or(0);+ return match apple_intelligence::process_text_with_system_prompt(+ &system_prompt,+ &user_content,+ token_limit,+ ) {+ Ok(result) => {+ if result.trim().is_empty() {+ debug!("Apple Intelligence returned an empty response");+ None+ } else {+ let result = strip_invisible_chars(&result);+ debug!(+ "Apple Intelligence post-processing succeeded. Output length: {} chars",+ result.len()+ );+ Some(result)+ }+ }+ Err(err) => {+ error!("Apple Intelligence post-processing failed: {}", err);None- } else {- debug!(- "Apple Intelligence post-processing succeeded. Output length: {} chars",- result.len()- );- Some(result)}- }- Err(err) => {- error!("Apple Intelligence post-processing failed: {}", err);- None- }- };+ };+ }++ #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))]+ {+ debug!("Apple Intelligence provider selected on unsupported platform");+ return None;+ }}- #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))]+ // Define JSON schema for transcription output+ let json_schema = serde_json::json!({+ "type": "object",+ "properties": {+ (TRANSCRIPTION_FIELD): {+ "type": "string",+ "description": "The cleaned and processed transcription text"+ }+ },+ "required": [TRANSCRIPTION_FIELD],+ "additionalProperties": false+ });++ match crate::llm_client::send_chat_completion_with_schema(+ &provider,+ api_key.clone(),+ &model,+ user_content,+ Some(system_prompt),+ Some(json_schema),+ )+ .await{- debug!("Apple Intelligence provider selected on unsupported platform");- return None;+ Ok(Some(content)) => {+ // Parse the JSON response to extract the transcription field+ match serde_json::from_str::<serde_json::Value>(&content) {+ Ok(json) => {+ if let Some(transcription_value) =+ json.get(TRANSCRIPTION_FIELD).and_then(|t| t.as_str())+ {+ let result = strip_invisible_chars(transcription_value);+ debug!(+ "Structured output post-processing succeeded for provider '{}'. Output length: {} chars",+ provider.id,+ result.len()+ );+ return Some(result);+ } else {+ error!("Structured output response missing 'transcription' field");+ return Some(strip_invisible_chars(&content));+ }+ }+ Err(e) => {+ error!(+ "Failed to parse structured output JSON: {}. Returning raw content.",+ e+ );+ return Some(strip_invisible_chars(&content));+ }+ }+ }+ Ok(None) => {+ error!("LLM API response has no content");+ return None;+ }+ Err(e) => {+ warn!(+ "Structured output failed for provider '{}': {}. Falling back to legacy mode.",+ provider.id, e+ );+ // Fall through to legacy mode below+ }}}- let api_key = settings- .post_process_api_keys- .get(&provider.id)- .cloned()- .unwrap_or_default();+ // Legacy mode: Replace ${output} variable in the prompt with the actual text+ let processed_prompt = prompt.replace("${output}", transcription);+ debug!("Processed prompt length: {} chars", processed_prompt.len());- // Send the chat completion requestmatch crate::llm_client::send_chat_completion(&provider, api_key, &model, processed_prompt).await{Ok(Some(content)) => {- // Strip invisible Unicode characters that some LLMs (e.g., Qwen) may insert- let content = content- .replace('\u{200B}', "") // Zero-Width Space- .replace('\u{200C}', "") // Zero-Width Non-Joiner- .replace('\u{200D}', "") // Zero-Width Joiner- .replace('\u{FEFF}', ""); // Byte Order Mark / Zero-Width No-Break Space+ let content = strip_invisible_chars(&content);debug!("LLM post-processing succeeded for provider '{}'. Output length: {} chars",provider.id,diff --git a/src-tauri/src/apple_intelligence.rs b/src-tauri/src/apple_intelligence.rsindex e34be15f..753ddce2 100644--- a/src-tauri/src/apple_intelligence.rs+++ b/src-tauri/src/apple_intelligence.rs@@ -12,10 +12,6 @@ pub struct AppleLLMResponse {// Link to the Swift functionsextern "C" {pub fn is_apple_intelligence_available() -> c_int;- pub fn process_text_with_apple_llm(- prompt: *const c_char,- max_tokens: i32,- ) -> *mut AppleLLMResponse;pub fn free_apple_llm_response(response: *mut AppleLLMResponse);}@@ -24,10 +20,27 @@ pub fn check_apple_intelligence_availability() -> bool {unsafe { is_apple_intelligence_available() == 1 }}-pub fn process_text(prompt: &str, max_tokens: i32) -> Result<String, String> {- let prompt_cstr = CString::new(prompt).map_err(|e| e.to_string())?;+// Link to the Swift function for system prompt support+extern "C" {+ pub fn process_text_with_system_prompt_apple(+ system_prompt: *const c_char,+ user_content: *const c_char,+ max_tokens: i32,+ ) -> *mut AppleLLMResponse;+}++/// Process text with Apple Intelligence using separate system prompt and user content+pub fn process_text_with_system_prompt(+ system_prompt: &str,+ user_content: &str,+ max_tokens: i32,+) -> Result<String, String> {+ let system_cstr = CString::new(system_prompt).map_err(|e| e.to_string())?;+ let user_cstr = CString::new(user_content).map_err(|e| e.to_string())?;- let response_ptr = unsafe { process_text_with_apple_llm(prompt_cstr.as_ptr(), max_tokens) };+ let response_ptr = unsafe {+ process_text_with_system_prompt_apple(system_cstr.as_ptr(), user_cstr.as_ptr(), max_tokens)+ };if response_ptr.is_null() {return Err("Null response from Apple LLM".to_string());diff --git a/src-tauri/src/llm_client.rs b/src-tauri/src/llm_client.rsindex 57fe67e2..2c8e17a7 100644--- a/src-tauri/src/llm_client.rs+++ b/src-tauri/src/llm_client.rs@@ -2,6 +2,7 @@ use crate::settings::PostProcessProvider;use log::debug;use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION, CONTENT_TYPE, REFERER, USER_AGENT};use serde::{Deserialize, Serialize};+use serde_json::Value;#[derive(Debug, Serialize)]struct ChatMessage {@@ -9,10 +10,26 @@ struct ChatMessage {content: String,}+#[derive(Debug, Serialize)]+struct JsonSchema {+ name: String,+ strict: bool,+ schema: Value,+}++#[derive(Debug, Serialize)]+struct ResponseFormat {+ #[serde(rename = "type")]+ format_type: String,+ json_schema: JsonSchema,+}+#[derive(Debug, Serialize)]struct ChatCompletionRequest {model: String,messages: Vec<ChatMessage>,+ #[serde(skip_serializing_if = "Option::is_none")]+ response_format: Option<ResponseFormat>,}#[derive(Debug, Deserialize)]@@ -84,6 +101,20 @@ pub async fn send_chat_completion(api_key: String,model: &str,prompt: String,+) -> Result<Option<String>, String> {+ send_chat_completion_with_schema(provider, api_key, model, prompt, None, None).await+}++/// Send a chat completion request with structured output support+/// When json_schema is provided, uses structured outputs mode+/// system_prompt is used as the system message when provided+pub async fn send_chat_completion_with_schema(+ provider: &PostProcessProvider,+ api_key: String,+ model: &str,+ user_content: String,+ system_prompt: Option<String>,+ json_schema: Option<Value>,) -> Result<Option<String>, String> {let base_url = provider.base_url.trim_end_matches('/');let url = format!("{}/chat/completions", base_url);@@ -92,12 +123,37 @@ pub async fn send_chat_completion(let client = create_client(provider, &api_key)?;+ // Build messages vector+ let mut messages = Vec::new();++ // Add system prompt if provided+ if let Some(system) = system_prompt {+ messages.push(ChatMessage {+ role: "system".to_string(),+ content: system,+ });+ }++ // Add user message+ messages.push(ChatMessage {+ role: "user".to_string(),+ content: user_content,+ });++ // Build response_format if schema is provided+ let response_format = json_schema.map(|schema| ResponseFormat {+ format_type: "json_schema".to_string(),+ json_schema: JsonSchema {+ name: "transcription_output".to_string(),+ strict: true,+ schema,+ },+ });+let request_body = ChatCompletionRequest {model: model.to_string(),- messages: vec![ChatMessage {- role: "user".to_string(),- content: prompt,- }],+ messages,+ response_format,};let response = clientdiff --git a/src-tauri/src/settings.rs b/src-tauri/src/settings.rsindex b59a4d7c..9468a84f 100644--- a/src-tauri/src/settings.rs+++ b/src-tauri/src/settings.rs@@ -101,6 +101,8 @@ pub struct PostProcessProvider {pub allow_base_url_edit: bool,#[serde(default)]pub models_endpoint: Option<String>,+ #[serde(default)]+ pub supports_structured_output: bool,}#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Type)]@@ -455,6 +457,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {base_url: "https://api.openai.com/v1".to_string(),allow_base_url_edit: false,models_endpoint: Some("/models".to_string()),+ supports_structured_output: true,},PostProcessProvider {id: "openrouter".to_string(),@@ -462,6 +465,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {base_url: "https://openrouter.ai/api/v1".to_string(),allow_base_url_edit: false,models_endpoint: Some("/models".to_string()),+ supports_structured_output: true,},PostProcessProvider {id: "anthropic".to_string(),@@ -469,6 +473,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {base_url: "https://api.anthropic.com/v1".to_string(),allow_base_url_edit: false,models_endpoint: Some("/models".to_string()),+ supports_structured_output: false,},PostProcessProvider {id: "groq".to_string(),@@ -476,6 +481,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {base_url: "https://api.groq.com/openai/v1".to_string(),allow_base_url_edit: false,models_endpoint: Some("/models".to_string()),+ supports_structured_output: false,},PostProcessProvider {id: "cerebras".to_string(),@@ -483,6 +489,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {base_url: "https://api.cerebras.ai/v1".to_string(),allow_base_url_edit: false,models_endpoint: Some("/models".to_string()),+ supports_structured_output: true,},];@@ -498,6 +505,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {base_url: "apple-intelligence://local".to_string(),allow_base_url_edit: false,models_endpoint: None,+ supports_structured_output: true,});}@@ -508,6 +516,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {base_url: "http://localhost:11434/v1".to_string(),allow_base_url_edit: true,models_endpoint: Some("/models".to_string()),+ supports_structured_output: false,});providers@@ -554,13 +563,30 @@ fn default_typing_tool() -> TypingTool {fn ensure_post_process_defaults(settings: &mut AppSettings) -> bool {let mut changed = false;for provider in default_post_process_providers() {- if settings+ // Use match to do a single lookup - either sync existing or add new+ match settings.post_process_providers- .iter()- .all(|existing| existing.id != provider.id)+ .iter_mut()+ .find(|p| p.id == provider.id){- settings.post_process_providers.push(provider.clone());- changed = true;+ Some(existing) => {+ // Sync supports_structured_output field for existing providers (migration)+ if existing.supports_structured_output != provider.supports_structured_output {+ debug!(+ "Updating supports_structured_output for provider '{}' from {} to {}",+ provider.id,+ existing.supports_structured_output,+ provider.supports_structured_output+ );+ existing.supports_structured_output = provider.supports_structured_output;+ changed = true;+ }+ }+ None => {+ // Provider doesn't exist, add it+ settings.post_process_providers.push(provider.clone());+ changed = true;+ }}if !settings.post_process_api_keys.contains_key(&provider.id) {diff --git a/src-tauri/swift/apple_intelligence.swift b/src-tauri/swift/apple_intelligence.swiftindex 98301371..df0dfda7 100644--- a/src-tauri/swift/apple_intelligence.swift+++ b/src-tauri/swift/apple_intelligence.swift@@ -50,12 +50,14 @@ public func isAppleIntelligenceAvailable() -> Int32 {}}-@_cdecl("process_text_with_apple_llm")-public func processTextWithAppleLLM(- _ prompt: UnsafePointer<CChar>,+@_cdecl("process_text_with_system_prompt_apple")+public func processTextWithSystemPrompt(+ _ systemPrompt: UnsafePointer<CChar>,+ _ userContent: UnsafePointer<CChar>,maxTokens: Int32) -> UnsafeMutablePointer<AppleLLMResponse> {- let swiftPrompt = String(cString: prompt)+ let swiftSystemPrompt = String(cString: systemPrompt)+ let swiftUserContent = String(cString: userContent)let responsePtr = ResponsePointer.allocate(capacity: 1)responsePtr.initialize(to: AppleLLMResponse(response: nil, success: 0, error_message: nil))@@ -87,17 +89,20 @@ public func processTextWithAppleLLM(Task.detached(priority: .userInitiated) {defer { semaphore.signal() }do {- let session = LanguageModelSession(model: model)+ let session = LanguageModelSession(+ model: model,+ instructions: swiftSystemPrompt+ )var output: Stringdo {let structured = try await session.respond(- to: swiftPrompt,+ to: swiftUserContent,generating: CleanedTranscript.self)output = structured.content.cleanedText} catch {- let fallbackGeneration = try await session.respond(to: swiftPrompt)+ let fallbackGeneration = try await session.respond(to: swiftUserContent)output = fallbackGeneration.content}diff --git a/src-tauri/swift/apple_intelligence_bridge.h b/src-tauri/swift/apple_intelligence_bridge.hindex a5c1ab0e..3adffce7 100644--- a/src-tauri/swift/apple_intelligence_bridge.h+++ b/src-tauri/swift/apple_intelligence_bridge.h@@ -16,8 +16,8 @@ typedef struct {// Check if Apple Intelligence is available on the deviceint is_apple_intelligence_available(void);-// Process text using Apple's on-device LLM-AppleLLMResponse* process_text_with_apple_llm(const char* prompt, int max_tokens);+// Process text using Apple's on-device LLM with separate system prompt and user content+AppleLLMResponse* process_text_with_system_prompt_apple(const char* system_prompt, const char* user_content, int max_tokens);// Free memory allocated by the Apple LLM responsevoid free_apple_llm_response(AppleLLMResponse* response);diff --git a/src-tauri/swift/apple_intelligence_stub.swift b/src-tauri/swift/apple_intelligence_stub.swiftindex 205a83e9..4ae0818e 100644--- a/src-tauri/swift/apple_intelligence_stub.swift+++ b/src-tauri/swift/apple_intelligence_stub.swift@@ -11,9 +11,10 @@ public func isAppleIntelligenceAvailable() -> Int32 {return 0}-@_cdecl("process_text_with_apple_llm")-public func processTextWithAppleLLM(- _ prompt: UnsafePointer<CChar>,+@_cdecl("process_text_with_system_prompt_apple")+public func processTextWithSystemPrompt(+ _ systemPrompt: UnsafePointer<CChar>,+ _ userContent: UnsafePointer<CChar>,maxTokens: Int32) -> UnsafeMutablePointer<AppleLLMResponse> {let responsePtr = ResponsePointer.allocate(capacity: 1)diff --git a/src/bindings.ts b/src/bindings.tsindex d85b915a..82c56cce 100644--- a/src/bindings.ts+++ b/src/bindings.ts@@ -754,7 +754,7 @@ export type ModelLoadStatus = { is_loaded: boolean; current_model: string | nullexport type ModelUnloadTimeout = "never" | "immediately" | "min_2" | "min_5" | "min_10" | "min_15" | "hour_1" | "sec_5"export type OverlayPosition = "none" | "top" | "bottom"export type PasteMethod = "ctrl_v" | "direct" | "none" | "shift_insert" | "ctrl_shift_v"-export type PostProcessProvider = { id: string; label: string; base_url: string; allow_base_url_edit?: boolean; models_endpoint?: string | null }+export type PostProcessProvider = { id: string; label: string; base_url: string; allow_base_url_edit?: boolean; models_endpoint?: string | null; supports_structured_output?: boolean }export type RecordingRetentionPeriod = "never" | "preserve_limit" | "days_3" | "weeks_2" | "months_3"export type ShortcutBinding = { id: string; name: string; description: string; default_binding: string; current_binding: string }export type SoundTheme = "marimba" | "pop" | "custom"