A free, open source, and extensible speech-to-text application that works completely offline.
retry without reasoning (#1809)
CJ Pais148e5492f43d
diff --git a/src-tauri/src/actions.rs b/src-tauri/src/actions.rsindex 0a428b5c..247eda42 100644--- a/src-tauri/src/actions.rs+++ b/src-tauri/src/actions.rs@@ -62,6 +62,19 @@ fn strip_invisible_chars(s: &str) -> String {s.replace(['\u{200B}', '\u{200C}', '\u{200D}', '\u{FEFF}'], "")}+/// Strip a leading `<think>...</think>` block. Some endpoints can't disable+/// reasoning, and some local servers put the reasoning text into `content`+/// instead of a separate field — without this the user would get the model's+/// chain of thought pasted along with the cleaned transcription.+fn strip_think_block(s: &str) -> &str {+ if let Some(rest) = s.trim_start().strip_prefix("<think>") {+ if let Some(end) = rest.find("</think>") {+ return rest[end + "</think>".len()..].trim_start();+ }+ }+ s+}+/// Build a system prompt from the user's prompt template./// Removes `${output}` placeholder since the transcription is sent as the user message.fn build_system_prompt(prompt_template: &str) -> String {@@ -168,21 +181,10 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str).cloned().unwrap_or_default();- // Disable reasoning for providers where post-processing rarely benefits from it.- // - custom: top-level reasoning_effort (works for local OpenAI-compat servers)- // - openrouter: nested reasoning object; exclude:true also keeps reasoning text- // out of the response so it can't pollute structured-output JSON parsing- let (reasoning_effort, reasoning) = match provider.id.as_str() {- "custom" => (Some("none".to_string()), None),- "openrouter" => (- None,- Some(crate::llm_client::ReasoningConfig {- effort: Some("none".to_string()),- exclude: Some(true),- }),- ),- _ => (None, None),- };+ // Ask these providers to skip reasoning/thinking — post-processing rarely+ // benefits from it and it adds seconds of latency. llm_client picks the+ // field the endpoint understands and retries without it if rejected.+ let disable_reasoning = matches!(provider.id.as_str(), "custom" | "openrouter");if provider.supports_structured_output {debug!("Using structured outputs for provider '{}'", provider.id);@@ -254,14 +256,14 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str)user_content,Some(system_prompt),Some(json_schema),- reasoning_effort.clone(),- reasoning.clone(),+ disable_reasoning,).await{Ok(Some(content)) => {// Parse the JSON response to extract the transcription field- match serde_json::from_str::<serde_json::Value>(&content) {+ let content = strip_think_block(&content);+ match serde_json::from_str::<serde_json::Value>(content) {Ok(json) => {if let Some(transcription_value) =json.get(TRANSCRIPTION_FIELD).and_then(|t| t.as_str())@@ -275,7 +277,7 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str)return Some(result);} else {error!("Structured output response missing 'transcription' field");- return Some(strip_invisible_chars(&content));+ return Some(strip_invisible_chars(content));}}Err(e) => {@@ -283,7 +285,7 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str)"Failed to parse structured output JSON: {}. Returning raw content.",e);- return Some(strip_invisible_chars(&content));+ return Some(strip_invisible_chars(content));}}}@@ -310,13 +312,12 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str)api_key,&model,processed_prompt,- reasoning_effort,- reasoning,+ disable_reasoning,).await{Ok(Some(content)) => {- let content = strip_invisible_chars(&content);+ let content = strip_invisible_chars(strip_think_block(&content));debug!("LLM post-processing succeeded for provider '{}'. Output length: {} chars",provider.id,@@ -927,7 +928,10 @@ pub static ACTION_MAP: Lazy<HashMap<String, Arc<dyn ShortcutAction>>> = Lazy::ne#[cfg(test)]mod tests {- use super::{complete_unless_cancelled, is_blank_transcription, should_use_streaming_overlay};+ use super::{+ complete_unless_cancelled, is_blank_transcription, should_use_streaming_overlay,+ strip_think_block,+ };use crate::settings::OverlayStyle;use std::future;use std::sync::atomic::{AtomicBool, Ordering};@@ -976,6 +980,32 @@ mod tests {assert_eq!(result, None);}+ #[test]+ fn leading_think_block_is_stripped() {+ assert_eq!(+ strip_think_block("<think>pondering...</think>Cleaned text."),+ "Cleaned text."+ );+ assert_eq!(+ strip_think_block(" \n<think>multi\nline</think>\n Cleaned text."),+ "Cleaned text."+ );+ }++ #[test]+ fn content_without_think_block_is_unchanged() {+ assert_eq!(strip_think_block("Cleaned text."), "Cleaned text.");+ assert_eq!(+ strip_think_block("Mentions <think> mid-sentence."),+ "Mentions <think> mid-sentence."+ );+ // Unclosed block: leave untouched rather than guess+ assert_eq!(+ strip_think_block("<think>never closed"),+ "<think>never closed"+ );+ }+#[test]fn live_overlay_uses_streaming_states_only_for_streaming_models() {assert!(should_use_streaming_overlay(OverlayStyle::Live, true));diff --git a/src-tauri/src/llm_client.rs b/src-tauri/src/llm_client.rsindex 2a06e173..6ed64c12 100644--- a/src-tauri/src/llm_client.rs+++ b/src-tauri/src/llm_client.rs@@ -1,8 +1,10 @@use crate::settings::PostProcessProvider;-use log::debug;+use log::{debug, info};use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION, CONTENT_TYPE, REFERER, USER_AGENT};use serde::{Deserialize, Serialize};use serde_json::Value;+use std::collections::HashSet;+use std::sync::{Mutex, OnceLock};#[derive(Debug, Serialize)]struct ChatMessage {@@ -24,12 +26,86 @@ struct ResponseFormat {json_schema: JsonSchema,}-#[derive(Debug, Serialize, Clone, Default)]-pub struct ReasoningConfig {+#[derive(Debug, Serialize, Clone, Default, PartialEq)]+struct ReasoningConfig {#[serde(skip_serializing_if = "Option::is_none")]- pub effort: Option<String>,+ effort: Option<String>,#[serde(skip_serializing_if = "Option::is_none")]- pub exclude: Option<bool>,+ exclude: Option<bool>,+}++/// Request fields used to ask an endpoint to skip reasoning/thinking.+/// Providers disagree on the field name and accepted values, so at most one of+/// these is set per request (see `reasoning_disable_params`).+#[derive(Debug, Serialize, Clone, Default, PartialEq)]+struct ReasoningParams {+ #[serde(skip_serializing_if = "Option::is_none")]+ reasoning_effort: Option<String>,+ #[serde(skip_serializing_if = "Option::is_none")]+ reasoning: Option<ReasoningConfig>,+ #[serde(skip_serializing_if = "Option::is_none")]+ thinking: Option<Value>,+}++impl ReasoningParams {+ fn is_empty(&self) -> bool {+ self.reasoning_effort.is_none() && self.reasoning.is_none() && self.thinking.is_none()+ }+}++/// Pick the reasoning-disable request fields an endpoint understands.+/// Unknown endpoints get the common OpenAI-style field; if they reject it,+/// the request is retried without it (see `send_chat_completion_with_schema`).+fn reasoning_disable_params(provider: &PostProcessProvider) -> ReasoningParams {+ let base_url = provider.base_url.to_lowercase();+ if base_url.contains("api.deepseek.com") {+ // DeepSeek rejects reasoning_effort "none" and uses its own field:+ // https://api-docs.deepseek.com/guides/thinking_mode+ ReasoningParams {+ thinking: Some(serde_json::json!({ "type": "disabled" })),+ ..Default::default()+ }+ } else if provider.id == "openrouter" {+ // OpenRouter nested object; exclude:true also keeps reasoning text out+ // of the response so it can't pollute structured-output JSON parsing+ ReasoningParams {+ reasoning: Some(ReasoningConfig {+ effort: Some("none".to_string()),+ exclude: Some(true),+ }),+ ..Default::default()+ }+ } else {+ ReasoningParams {+ reasoning_effort: Some("none".to_string()),+ ..Default::default()+ }+ }+}++/// Endpoints (base_url|model) that rejected the reasoning-disable fields with a+/// 4xx. Remembered for the lifetime of the process so every dictation after the+/// first skips the doomed attempt and goes straight to a plain request.+fn reasoning_rejections() -> &'static Mutex<HashSet<String>> {+ static REJECTED: OnceLock<Mutex<HashSet<String>>> = OnceLock::new();+ REJECTED.get_or_init(|| Mutex::new(HashSet::new()))+}++fn endpoint_key(provider: &PostProcessProvider, model: &str) -> String {+ format!("{}|{}", provider.base_url.trim_end_matches('/'), model)+}++fn is_known_rejected(key: &str) -> bool {+ reasoning_rejections()+ .lock()+ .map(|set| set.contains(key))+ .unwrap_or(false)+}++fn remember_rejection(key: String) {+ if let Ok(mut set) = reasoning_rejections().lock() {+ set.insert(key);+ }}#[derive(Debug, Serialize)]@@ -38,10 +114,8 @@ struct ChatCompletionRequest {messages: Vec<ChatMessage>,#[serde(skip_serializing_if = "Option::is_none")]response_format: Option<ResponseFormat>,- #[serde(skip_serializing_if = "Option::is_none")]- reasoning_effort: Option<String>,- #[serde(skip_serializing_if = "Option::is_none")]- reasoning: Option<ReasoningConfig>,+ #[serde(flatten)]+ reasoning: ReasoningParams,}#[derive(Debug, Deserialize)]@@ -113,8 +187,7 @@ pub async fn send_chat_completion(api_key: String,model: &str,prompt: String,- reasoning_effort: Option<String>,- reasoning: Option<ReasoningConfig>,+ disable_reasoning: bool,) -> Result<Option<String>, String> {send_chat_completion_with_schema(provider,@@ -123,18 +196,21 @@ pub async fn send_chat_completion(prompt,None,None,- reasoning_effort,- reasoning,+ disable_reasoning,).await}-/// Send a chat completion request with structured output support-/// When json_schema is provided, uses structured outputs mode-/// system_prompt is used as the system message when provided-/// reasoning_effort sets the OpenAI-style top-level field (e.g., "none", "low", "medium", "high")-/// reasoning sets the OpenRouter-style nested object (effort + exclude)-#[allow(clippy::too_many_arguments)]+/// Send a chat completion request with structured output support.+/// When json_schema is provided, uses structured outputs mode.+/// system_prompt is used as the system message when provided.+///+/// When disable_reasoning is set, the request carries the reasoning-disable+/// fields the endpoint is expected to understand. Not every OpenAI-compatible+/// endpoint accepts them (DeepSeek, Gemini's compat layer, and some OpenRouter+/// upstreams reject with 400), so a 400/422 answer to such a request triggers+/// one retry without the fields, and the rejection is remembered per+/// (base_url, model) so later requests skip the failing attempt entirely.pub async fn send_chat_completion_with_schema(provider: &PostProcessProvider,api_key: String,@@ -142,8 +218,7 @@ pub async fn send_chat_completion_with_schema(user_content: String,system_prompt: Option<String>,json_schema: Option<Value>,- reasoning_effort: Option<String>,- reasoning: Option<ReasoningConfig>,+ disable_reasoning: bool,) -> Result<Option<String>, String> {let base_url = provider.base_url.trim_end_matches('/');let url = format!("{}/chat/completions", base_url);@@ -179,22 +254,61 @@ pub async fn send_chat_completion_with_schema(},});- let request_body = ChatCompletionRequest {+ let key = endpoint_key(provider, model);+ let reasoning = if disable_reasoning && !is_known_rejected(&key) {+ reasoning_disable_params(provider)+ } else {+ ReasoningParams::default()+ };++ let mut request_body = ChatCompletionRequest {model: model.to_string(),messages,response_format,- reasoning_effort,reasoning,};- let response = client+ let mut response = client.post(&url).json(&request_body).send().await.map_err(|e| format!("HTTP request failed: {}", e))?;+ let mut status = response.status();++ // A 400/422 on a request carrying reasoning-disable fields is almost always+ // the endpoint rejecting those fields — retry once without them.+ if !status.is_success()+ && matches!(status.as_u16(), 400 | 422)+ && !request_body.reasoning.is_empty()+ {+ let error_text = response+ .text()+ .await+ .unwrap_or_else(|_| "Failed to read error response".to_string());+ info!(+ "Endpoint rejected request with reasoning disabled (status {}): {}. Retrying without reasoning fields",+ status, error_text+ );++ request_body.reasoning = ReasoningParams::default();+ response = client+ .post(&url)+ .json(&request_body)+ .send()+ .await+ .map_err(|e| format!("HTTP request failed: {}", e))?;+ status = response.status();++ if status.is_success() {+ info!(+ "Retry without reasoning fields succeeded; '{}' (model '{}') will skip them from now on",+ base_url, model+ );+ remember_rejection(key);+ }+ }- let status = response.status();if !status.is_success() {let error_text = response.text()@@ -276,3 +390,96 @@ pub async fn fetch_models(Ok(models)}++#[cfg(test)]+mod tests {+ use super::*;++ fn provider(id: &str, base_url: &str) -> PostProcessProvider {+ PostProcessProvider {+ id: id.to_string(),+ label: id.to_string(),+ base_url: base_url.to_string(),+ allow_base_url_edit: true,+ models_endpoint: None,+ supports_structured_output: false,+ }+ }++ fn request_json(reasoning: ReasoningParams) -> Value {+ let request = ChatCompletionRequest {+ model: "test-model".to_string(),+ messages: vec![ChatMessage {+ role: "user".to_string(),+ content: "hi".to_string(),+ }],+ response_format: None,+ reasoning,+ };+ serde_json::to_value(&request).unwrap()+ }++ #[test]+ fn default_reasoning_params_serialize_to_no_fields() {+ let json = request_json(ReasoningParams::default());+ assert!(json.get("reasoning_effort").is_none());+ assert!(json.get("reasoning").is_none());+ assert!(json.get("thinking").is_none());+ }++ #[test]+ fn custom_provider_uses_top_level_reasoning_effort() {+ let params = reasoning_disable_params(&provider("custom", "http://localhost:11434/v1"));+ let json = request_json(params);+ assert_eq!(json["reasoning_effort"], "none");+ assert!(json.get("reasoning").is_none());+ assert!(json.get("thinking").is_none());+ }++ #[test]+ fn openrouter_uses_nested_reasoning_object() {+ let params =+ reasoning_disable_params(&provider("openrouter", "https://openrouter.ai/api/v1"));+ let json = request_json(params);+ assert!(json.get("reasoning_effort").is_none());+ assert_eq!(json["reasoning"]["effort"], "none");+ assert_eq!(json["reasoning"]["exclude"], true);+ assert!(json.get("thinking").is_none());+ }++ #[test]+ fn deepseek_base_url_uses_thinking_disabled() {+ let params = reasoning_disable_params(&provider("custom", "https://api.deepseek.com"));+ let json = request_json(params);+ assert!(json.get("reasoning_effort").is_none());+ assert!(json.get("reasoning").is_none());+ assert_eq!(json["thinking"]["type"], "disabled");+ }++ #[test]+ fn reasoning_params_is_empty_tracks_all_fields() {+ assert!(ReasoningParams::default().is_empty());+ assert!(!ReasoningParams {+ reasoning_effort: Some("none".to_string()),+ ..Default::default()+ }+ .is_empty());+ assert!(!ReasoningParams {+ thinking: Some(serde_json::json!({ "type": "disabled" })),+ ..Default::default()+ }+ .is_empty());+ }++ #[test]+ fn rejection_memo_is_keyed_by_base_url_and_model() {+ let deepseek = provider("custom", "https://api.deepseek.com/");+ let key = endpoint_key(&deepseek, "deepseek-chat");+ assert_eq!(key, "https://api.deepseek.com|deepseek-chat");+ assert!(!is_known_rejected(&key));+ remember_rejection(key.clone());+ assert!(is_known_rejected(&key));+ // A different model on the same endpoint is tracked separately+ assert!(!is_known_rejected(&endpoint_key(&deepseek, "other-model")));+ }+}