Skip to content

armgit / Handy

Public Imported from GitHub
0

A free, open source, and extensible speech-to-text application that works completely offline.

feat: implement structured outputs for post-processing providers (#706)

Chirag Agggarwal0cb8ab21624c
diff --git a/src-tauri/src/actions.rs b/src-tauri/src/actions.rs
index 53393881..f19ce58b 100644
--- a/src-tauri/src/actions.rs
+++ b/src-tauri/src/actions.rs
@@ -12,7 +12,7 @@ use crate::utils::{
};
use crate::TranscriptionCoordinator;
use ferrous_opencc::{config::BuiltinConfig, OpenCC};
-use log::{debug, error};
+use log::{debug, error, warn};
use once_cell::sync::Lazy;
use std::collections::HashMap;
use std::sync::Arc;
@@ -42,6 +42,20 @@ struct TranscribeAction {
post_process: bool,
}
+/// Field name for structured output JSON schema
+const TRANSCRIPTION_FIELD: &str = "transcription";
+
+/// Strip invisible Unicode characters that some LLMs may insert
+fn strip_invisible_chars(s: &str) -> String {
+ s.replace(['\u{200B}', '\u{200C}', '\u{200D}', '\u{FEFF}'], "")
+}
+
+/// Build a system prompt from the user's prompt template.
+/// Removes `${output}` placeholder since the transcription is sent as the user message.
+fn build_system_prompt(prompt_template: &str) -> String {
+ prompt_template.replace("${output}", "").trim().to_string()
+}
+
async fn post_process_transcription(settings: &AppSettings, transcription: &str) -> Option<String> {
let provider = match settings.active_post_process_provider().cloned() {
Some(provider) => provider,
@@ -98,63 +112,136 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str)
provider.id, model
);
- // Replace ${output} variable in the prompt with the actual text
- let processed_prompt = prompt.replace("${output}", transcription);
- debug!("Processed prompt length: {} chars", processed_prompt.len());
+ let api_key = settings
+ .post_process_api_keys
+ .get(&provider.id)
+ .cloned()
+ .unwrap_or_default();
- if provider.id == APPLE_INTELLIGENCE_PROVIDER_ID {
- #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
- {
- if !apple_intelligence::check_apple_intelligence_availability() {
- debug!("Apple Intelligence selected but not currently available on this device");
- return None;
- }
+ if provider.supports_structured_output {
+ debug!("Using structured outputs for provider '{}'", provider.id);
+
+ let system_prompt = build_system_prompt(&prompt);
+ let user_content = transcription.to_string();
+
+ // Handle Apple Intelligence separately since it uses native Swift APIs
+ if provider.id == APPLE_INTELLIGENCE_PROVIDER_ID {
+ #[cfg(all(target_os = "macos", target_arch = "aarch64"))]
+ {
+ if !apple_intelligence::check_apple_intelligence_availability() {
+ debug!(
+ "Apple Intelligence selected but not currently available on this device"
+ );
+ return None;
+ }
- let token_limit = model.trim().parse::<i32>().unwrap_or(0);
- return match apple_intelligence::process_text(&processed_prompt, token_limit) {
- Ok(result) => {
- if result.trim().is_empty() {
- debug!("Apple Intelligence returned an empty response");
+ let token_limit = model.trim().parse::<i32>().unwrap_or(0);
+ return match apple_intelligence::process_text_with_system_prompt(
+ &system_prompt,
+ &user_content,
+ token_limit,
+ ) {
+ Ok(result) => {
+ if result.trim().is_empty() {
+ debug!("Apple Intelligence returned an empty response");
+ None
+ } else {
+ let result = strip_invisible_chars(&result);
+ debug!(
+ "Apple Intelligence post-processing succeeded. Output length: {} chars",
+ result.len()
+ );
+ Some(result)
+ }
+ }
+ Err(err) => {
+ error!("Apple Intelligence post-processing failed: {}", err);
None
- } else {
- debug!(
- "Apple Intelligence post-processing succeeded. Output length: {} chars",
- result.len()
- );
- Some(result)
}
- }
- Err(err) => {
- error!("Apple Intelligence post-processing failed: {}", err);
- None
- }
- };
+ };
+ }
+
+ #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))]
+ {
+ debug!("Apple Intelligence provider selected on unsupported platform");
+ return None;
+ }
}
- #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))]
+ // Define JSON schema for transcription output
+ let json_schema = serde_json::json!({
+ "type": "object",
+ "properties": {
+ (TRANSCRIPTION_FIELD): {
+ "type": "string",
+ "description": "The cleaned and processed transcription text"
+ }
+ },
+ "required": [TRANSCRIPTION_FIELD],
+ "additionalProperties": false
+ });
+
+ match crate::llm_client::send_chat_completion_with_schema(
+ &provider,
+ api_key.clone(),
+ &model,
+ user_content,
+ Some(system_prompt),
+ Some(json_schema),
+ )
+ .await
{
- debug!("Apple Intelligence provider selected on unsupported platform");
- return None;
+ Ok(Some(content)) => {
+ // Parse the JSON response to extract the transcription field
+ match serde_json::from_str::<serde_json::Value>(&content) {
+ Ok(json) => {
+ if let Some(transcription_value) =
+ json.get(TRANSCRIPTION_FIELD).and_then(|t| t.as_str())
+ {
+ let result = strip_invisible_chars(transcription_value);
+ debug!(
+ "Structured output post-processing succeeded for provider '{}'. Output length: {} chars",
+ provider.id,
+ result.len()
+ );
+ return Some(result);
+ } else {
+ error!("Structured output response missing 'transcription' field");
+ return Some(strip_invisible_chars(&content));
+ }
+ }
+ Err(e) => {
+ error!(
+ "Failed to parse structured output JSON: {}. Returning raw content.",
+ e
+ );
+ return Some(strip_invisible_chars(&content));
+ }
+ }
+ }
+ Ok(None) => {
+ error!("LLM API response has no content");
+ return None;
+ }
+ Err(e) => {
+ warn!(
+ "Structured output failed for provider '{}': {}. Falling back to legacy mode.",
+ provider.id, e
+ );
+ // Fall through to legacy mode below
+ }
}
}
- let api_key = settings
- .post_process_api_keys
- .get(&provider.id)
- .cloned()
- .unwrap_or_default();
+ // Legacy mode: Replace ${output} variable in the prompt with the actual text
+ let processed_prompt = prompt.replace("${output}", transcription);
+ debug!("Processed prompt length: {} chars", processed_prompt.len());
- // Send the chat completion request
match crate::llm_client::send_chat_completion(&provider, api_key, &model, processed_prompt)
.await
{
Ok(Some(content)) => {
- // Strip invisible Unicode characters that some LLMs (e.g., Qwen) may insert
- let content = content
- .replace('\u{200B}', "") // Zero-Width Space
- .replace('\u{200C}', "") // Zero-Width Non-Joiner
- .replace('\u{200D}', "") // Zero-Width Joiner
- .replace('\u{FEFF}', ""); // Byte Order Mark / Zero-Width No-Break Space
+ let content = strip_invisible_chars(&content);
debug!(
"LLM post-processing succeeded for provider '{}'. Output length: {} chars",
provider.id,
diff --git a/src-tauri/src/apple_intelligence.rs b/src-tauri/src/apple_intelligence.rs
index e34be15f..753ddce2 100644
--- a/src-tauri/src/apple_intelligence.rs
+++ b/src-tauri/src/apple_intelligence.rs
@@ -12,10 +12,6 @@ pub struct AppleLLMResponse {
// Link to the Swift functions
extern "C" {
pub fn is_apple_intelligence_available() -> c_int;
- pub fn process_text_with_apple_llm(
- prompt: *const c_char,
- max_tokens: i32,
- ) -> *mut AppleLLMResponse;
pub fn free_apple_llm_response(response: *mut AppleLLMResponse);
}
@@ -24,10 +20,27 @@ pub fn check_apple_intelligence_availability() -> bool {
unsafe { is_apple_intelligence_available() == 1 }
}
-pub fn process_text(prompt: &str, max_tokens: i32) -> Result<String, String> {
- let prompt_cstr = CString::new(prompt).map_err(|e| e.to_string())?;
+// Link to the Swift function for system prompt support
+extern "C" {
+ pub fn process_text_with_system_prompt_apple(
+ system_prompt: *const c_char,
+ user_content: *const c_char,
+ max_tokens: i32,
+ ) -> *mut AppleLLMResponse;
+}
+
+/// Process text with Apple Intelligence using separate system prompt and user content
+pub fn process_text_with_system_prompt(
+ system_prompt: &str,
+ user_content: &str,
+ max_tokens: i32,
+) -> Result<String, String> {
+ let system_cstr = CString::new(system_prompt).map_err(|e| e.to_string())?;
+ let user_cstr = CString::new(user_content).map_err(|e| e.to_string())?;
- let response_ptr = unsafe { process_text_with_apple_llm(prompt_cstr.as_ptr(), max_tokens) };
+ let response_ptr = unsafe {
+ process_text_with_system_prompt_apple(system_cstr.as_ptr(), user_cstr.as_ptr(), max_tokens)
+ };
if response_ptr.is_null() {
return Err("Null response from Apple LLM".to_string());
diff --git a/src-tauri/src/llm_client.rs b/src-tauri/src/llm_client.rs
index 57fe67e2..2c8e17a7 100644
--- a/src-tauri/src/llm_client.rs
+++ b/src-tauri/src/llm_client.rs
@@ -2,6 +2,7 @@ use crate::settings::PostProcessProvider;
use log::debug;
use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION, CONTENT_TYPE, REFERER, USER_AGENT};
use serde::{Deserialize, Serialize};
+use serde_json::Value;
#[derive(Debug, Serialize)]
struct ChatMessage {
@@ -9,10 +10,26 @@ struct ChatMessage {
content: String,
}
+#[derive(Debug, Serialize)]
+struct JsonSchema {
+ name: String,
+ strict: bool,
+ schema: Value,
+}
+
+#[derive(Debug, Serialize)]
+struct ResponseFormat {
+ #[serde(rename = "type")]
+ format_type: String,
+ json_schema: JsonSchema,
+}
+
#[derive(Debug, Serialize)]
struct ChatCompletionRequest {
model: String,
messages: Vec<ChatMessage>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ response_format: Option<ResponseFormat>,
}
#[derive(Debug, Deserialize)]
@@ -84,6 +101,20 @@ pub async fn send_chat_completion(
api_key: String,
model: &str,
prompt: String,
+) -> Result<Option<String>, String> {
+ send_chat_completion_with_schema(provider, api_key, model, prompt, None, None).await
+}
+
+/// Send a chat completion request with structured output support
+/// When json_schema is provided, uses structured outputs mode
+/// system_prompt is used as the system message when provided
+pub async fn send_chat_completion_with_schema(
+ provider: &PostProcessProvider,
+ api_key: String,
+ model: &str,
+ user_content: String,
+ system_prompt: Option<String>,
+ json_schema: Option<Value>,
) -> Result<Option<String>, String> {
let base_url = provider.base_url.trim_end_matches('/');
let url = format!("{}/chat/completions", base_url);
@@ -92,12 +123,37 @@ pub async fn send_chat_completion(
let client = create_client(provider, &api_key)?;
+ // Build messages vector
+ let mut messages = Vec::new();
+
+ // Add system prompt if provided
+ if let Some(system) = system_prompt {
+ messages.push(ChatMessage {
+ role: "system".to_string(),
+ content: system,
+ });
+ }
+
+ // Add user message
+ messages.push(ChatMessage {
+ role: "user".to_string(),
+ content: user_content,
+ });
+
+ // Build response_format if schema is provided
+ let response_format = json_schema.map(|schema| ResponseFormat {
+ format_type: "json_schema".to_string(),
+ json_schema: JsonSchema {
+ name: "transcription_output".to_string(),
+ strict: true,
+ schema,
+ },
+ });
+
let request_body = ChatCompletionRequest {
model: model.to_string(),
- messages: vec![ChatMessage {
- role: "user".to_string(),
- content: prompt,
- }],
+ messages,
+ response_format,
};
let response = client
diff --git a/src-tauri/src/settings.rs b/src-tauri/src/settings.rs
index b59a4d7c..9468a84f 100644
--- a/src-tauri/src/settings.rs
+++ b/src-tauri/src/settings.rs
@@ -101,6 +101,8 @@ pub struct PostProcessProvider {
pub allow_base_url_edit: bool,
#[serde(default)]
pub models_endpoint: Option<String>,
+ #[serde(default)]
+ pub supports_structured_output: bool,
}
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Type)]
@@ -455,6 +457,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {
base_url: "https://api.openai.com/v1".to_string(),
allow_base_url_edit: false,
models_endpoint: Some("/models".to_string()),
+ supports_structured_output: true,
},
PostProcessProvider {
id: "openrouter".to_string(),
@@ -462,6 +465,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {
base_url: "https://openrouter.ai/api/v1".to_string(),
allow_base_url_edit: false,
models_endpoint: Some("/models".to_string()),
+ supports_structured_output: true,
},
PostProcessProvider {
id: "anthropic".to_string(),
@@ -469,6 +473,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {
base_url: "https://api.anthropic.com/v1".to_string(),
allow_base_url_edit: false,
models_endpoint: Some("/models".to_string()),
+ supports_structured_output: false,
},
PostProcessProvider {
id: "groq".to_string(),
@@ -476,6 +481,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {
base_url: "https://api.groq.com/openai/v1".to_string(),
allow_base_url_edit: false,
models_endpoint: Some("/models".to_string()),
+ supports_structured_output: false,
},
PostProcessProvider {
id: "cerebras".to_string(),
@@ -483,6 +489,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {
base_url: "https://api.cerebras.ai/v1".to_string(),
allow_base_url_edit: false,
models_endpoint: Some("/models".to_string()),
+ supports_structured_output: true,
},
];
@@ -498,6 +505,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {
base_url: "apple-intelligence://local".to_string(),
allow_base_url_edit: false,
models_endpoint: None,
+ supports_structured_output: true,
});
}
@@ -508,6 +516,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {
base_url: "http://localhost:11434/v1".to_string(),
allow_base_url_edit: true,
models_endpoint: Some("/models".to_string()),
+ supports_structured_output: false,
});
providers
@@ -554,13 +563,30 @@ fn default_typing_tool() -> TypingTool {
fn ensure_post_process_defaults(settings: &mut AppSettings) -> bool {
let mut changed = false;
for provider in default_post_process_providers() {
- if settings
+ // Use match to do a single lookup - either sync existing or add new
+ match settings
.post_process_providers
- .iter()
- .all(|existing| existing.id != provider.id)
+ .iter_mut()
+ .find(|p| p.id == provider.id)
{
- settings.post_process_providers.push(provider.clone());
- changed = true;
+ Some(existing) => {
+ // Sync supports_structured_output field for existing providers (migration)
+ if existing.supports_structured_output != provider.supports_structured_output {
+ debug!(
+ "Updating supports_structured_output for provider '{}' from {} to {}",
+ provider.id,
+ existing.supports_structured_output,
+ provider.supports_structured_output
+ );
+ existing.supports_structured_output = provider.supports_structured_output;
+ changed = true;
+ }
+ }
+ None => {
+ // Provider doesn't exist, add it
+ settings.post_process_providers.push(provider.clone());
+ changed = true;
+ }
}
if !settings.post_process_api_keys.contains_key(&provider.id) {
diff --git a/src-tauri/swift/apple_intelligence.swift b/src-tauri/swift/apple_intelligence.swift
index 98301371..df0dfda7 100644
--- a/src-tauri/swift/apple_intelligence.swift
+++ b/src-tauri/swift/apple_intelligence.swift
@@ -50,12 +50,14 @@ public func isAppleIntelligenceAvailable() -> Int32 {
}
}
-@_cdecl("process_text_with_apple_llm")
-public func processTextWithAppleLLM(
- _ prompt: UnsafePointer<CChar>,
+@_cdecl("process_text_with_system_prompt_apple")
+public func processTextWithSystemPrompt(
+ _ systemPrompt: UnsafePointer<CChar>,
+ _ userContent: UnsafePointer<CChar>,
maxTokens: Int32
) -> UnsafeMutablePointer<AppleLLMResponse> {
- let swiftPrompt = String(cString: prompt)
+ let swiftSystemPrompt = String(cString: systemPrompt)
+ let swiftUserContent = String(cString: userContent)
let responsePtr = ResponsePointer.allocate(capacity: 1)
responsePtr.initialize(to: AppleLLMResponse(response: nil, success: 0, error_message: nil))
@@ -87,17 +89,20 @@ public func processTextWithAppleLLM(
Task.detached(priority: .userInitiated) {
defer { semaphore.signal() }
do {
- let session = LanguageModelSession(model: model)
+ let session = LanguageModelSession(
+ model: model,
+ instructions: swiftSystemPrompt
+ )
var output: String
do {
let structured = try await session.respond(
- to: swiftPrompt,
+ to: swiftUserContent,
generating: CleanedTranscript.self
)
output = structured.content.cleanedText
} catch {
- let fallbackGeneration = try await session.respond(to: swiftPrompt)
+ let fallbackGeneration = try await session.respond(to: swiftUserContent)
output = fallbackGeneration.content
}
diff --git a/src-tauri/swift/apple_intelligence_bridge.h b/src-tauri/swift/apple_intelligence_bridge.h
index a5c1ab0e..3adffce7 100644
--- a/src-tauri/swift/apple_intelligence_bridge.h
+++ b/src-tauri/swift/apple_intelligence_bridge.h
@@ -16,8 +16,8 @@ typedef struct {
// Check if Apple Intelligence is available on the device
int is_apple_intelligence_available(void);
-// Process text using Apple's on-device LLM
-AppleLLMResponse* process_text_with_apple_llm(const char* prompt, int max_tokens);
+// Process text using Apple's on-device LLM with separate system prompt and user content
+AppleLLMResponse* process_text_with_system_prompt_apple(const char* system_prompt, const char* user_content, int max_tokens);
// Free memory allocated by the Apple LLM response
void free_apple_llm_response(AppleLLMResponse* response);
diff --git a/src-tauri/swift/apple_intelligence_stub.swift b/src-tauri/swift/apple_intelligence_stub.swift
index 205a83e9..4ae0818e 100644
--- a/src-tauri/swift/apple_intelligence_stub.swift
+++ b/src-tauri/swift/apple_intelligence_stub.swift
@@ -11,9 +11,10 @@ public func isAppleIntelligenceAvailable() -> Int32 {
return 0
}
-@_cdecl("process_text_with_apple_llm")
-public func processTextWithAppleLLM(
- _ prompt: UnsafePointer<CChar>,
+@_cdecl("process_text_with_system_prompt_apple")
+public func processTextWithSystemPrompt(
+ _ systemPrompt: UnsafePointer<CChar>,
+ _ userContent: UnsafePointer<CChar>,
maxTokens: Int32
) -> UnsafeMutablePointer<AppleLLMResponse> {
let responsePtr = ResponsePointer.allocate(capacity: 1)
diff --git a/src/bindings.ts b/src/bindings.ts
index d85b915a..82c56cce 100644
--- a/src/bindings.ts
+++ b/src/bindings.ts
@@ -754,7 +754,7 @@ export type ModelLoadStatus = { is_loaded: boolean; current_model: string | null
export type ModelUnloadTimeout = "never" | "immediately" | "min_2" | "min_5" | "min_10" | "min_15" | "hour_1" | "sec_5"
export type OverlayPosition = "none" | "top" | "bottom"
export type PasteMethod = "ctrl_v" | "direct" | "none" | "shift_insert" | "ctrl_shift_v"
-export type PostProcessProvider = { id: string; label: string; base_url: string; allow_base_url_edit?: boolean; models_endpoint?: string | null }
+export type PostProcessProvider = { id: string; label: string; base_url: string; allow_base_url_edit?: boolean; models_endpoint?: string | null; supports_structured_output?: boolean }
export type RecordingRetentionPeriod = "never" | "preserve_limit" | "days_3" | "weeks_2" | "months_3"
export type ShortcutBinding = { id: string; name: string; description: string; default_binding: string; current_binding: string }
export type SoundTheme = "marimba" | "pop" | "custom"