Skip to content

armgit / Handy

Public Imported from GitHub
0

A free, open source, and extensible speech-to-text application that works completely offline.

perf: add reasoning_effort passthrough to avoid thinking-mode latency in local models (#1221)

LuoXi84d88f911c80
diff --git a/src-tauri/src/actions.rs b/src-tauri/src/actions.rs
index 3219caf9..bf65667a 100644
--- a/src-tauri/src/actions.rs
+++ b/src-tauri/src/actions.rs
@@ -125,6 +125,22 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str)
.cloned()
.unwrap_or_default();
+ // Disable reasoning for providers where post-processing rarely benefits from it.
+ // - custom: top-level reasoning_effort (works for local OpenAI-compat servers)
+ // - openrouter: nested reasoning object; exclude:true also keeps reasoning text
+ // out of the response so it can't pollute structured-output JSON parsing
+ let (reasoning_effort, reasoning) = match provider.id.as_str() {
+ "custom" => (Some("none".to_string()), None),
+ "openrouter" => (
+ None,
+ Some(crate::llm_client::ReasoningConfig {
+ effort: Some("none".to_string()),
+ exclude: Some(true),
+ }),
+ ),
+ _ => (None, None),
+ };
+
if provider.supports_structured_output {
debug!("Using structured outputs for provider '{}'", provider.id);
@@ -195,6 +211,8 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str)
user_content,
Some(system_prompt),
Some(json_schema),
+ reasoning_effort.clone(),
+ reasoning.clone(),
)
.await
{
@@ -244,8 +262,15 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str)
let processed_prompt = prompt.replace("${output}", transcription);
debug!("Processed prompt length: {} chars", processed_prompt.len());
- match crate::llm_client::send_chat_completion(&provider, api_key, &model, processed_prompt)
- .await
+ match crate::llm_client::send_chat_completion(
+ &provider,
+ api_key,
+ &model,
+ processed_prompt,
+ reasoning_effort,
+ reasoning,
+ )
+ .await
{
Ok(Some(content)) => {
let content = strip_invisible_chars(&content);
diff --git a/src-tauri/src/llm_client.rs b/src-tauri/src/llm_client.rs
index 2c8e17a7..08102c6b 100644
--- a/src-tauri/src/llm_client.rs
+++ b/src-tauri/src/llm_client.rs
@@ -24,12 +24,24 @@ struct ResponseFormat {
json_schema: JsonSchema,
}
+#[derive(Debug, Serialize, Clone, Default)]
+pub struct ReasoningConfig {
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub effort: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub exclude: Option<bool>,
+}
+
#[derive(Debug, Serialize)]
struct ChatCompletionRequest {
model: String,
messages: Vec<ChatMessage>,
#[serde(skip_serializing_if = "Option::is_none")]
response_format: Option<ResponseFormat>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ reasoning_effort: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ reasoning: Option<ReasoningConfig>,
}
#[derive(Debug, Deserialize)]
@@ -101,13 +113,27 @@ pub async fn send_chat_completion(
api_key: String,
model: &str,
prompt: String,
+ reasoning_effort: Option<String>,
+ reasoning: Option<ReasoningConfig>,
) -> Result<Option<String>, String> {
- send_chat_completion_with_schema(provider, api_key, model, prompt, None, None).await
+ send_chat_completion_with_schema(
+ provider,
+ api_key,
+ model,
+ prompt,
+ None,
+ None,
+ reasoning_effort,
+ reasoning,
+ )
+ .await
}
/// Send a chat completion request with structured output support
/// When json_schema is provided, uses structured outputs mode
/// system_prompt is used as the system message when provided
+/// reasoning_effort sets the OpenAI-style top-level field (e.g., "none", "low", "medium", "high")
+/// reasoning sets the OpenRouter-style nested object (effort + exclude)
pub async fn send_chat_completion_with_schema(
provider: &PostProcessProvider,
api_key: String,
@@ -115,6 +141,8 @@ pub async fn send_chat_completion_with_schema(
user_content: String,
system_prompt: Option<String>,
json_schema: Option<Value>,
+ reasoning_effort: Option<String>,
+ reasoning: Option<ReasoningConfig>,
) -> Result<Option<String>, String> {
let base_url = provider.base_url.trim_end_matches('/');
let url = format!("{}/chat/completions", base_url);
@@ -154,6 +182,8 @@ pub async fn send_chat_completion_with_schema(
model: model.to_string(),
messages,
response_format,
+ reasoning_effort,
+ reasoning,
};
let response = client