Skip to content

armgit / Handy

Public Imported from GitHub
0

A free, open source, and extensible speech-to-text application that works completely offline.

retry without reasoning (#1809)

CJ Pais148e5492f43d
diff --git a/src-tauri/src/actions.rs b/src-tauri/src/actions.rs
index 0a428b5c..247eda42 100644
--- a/src-tauri/src/actions.rs
+++ b/src-tauri/src/actions.rs
@@ -62,6 +62,19 @@ fn strip_invisible_chars(s: &str) -> String {
s.replace(['\u{200B}', '\u{200C}', '\u{200D}', '\u{FEFF}'], "")
}
+/// Strip a leading `<think>...</think>` block. Some endpoints can't disable
+/// reasoning, and some local servers put the reasoning text into `content`
+/// instead of a separate field — without this the user would get the model's
+/// chain of thought pasted along with the cleaned transcription.
+fn strip_think_block(s: &str) -> &str {
+ if let Some(rest) = s.trim_start().strip_prefix("<think>") {
+ if let Some(end) = rest.find("</think>") {
+ return rest[end + "</think>".len()..].trim_start();
+ }
+ }
+ s
+}
+
/// Build a system prompt from the user's prompt template.
/// Removes `${output}` placeholder since the transcription is sent as the user message.
fn build_system_prompt(prompt_template: &str) -> String {
@@ -168,21 +181,10 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str)
.cloned()
.unwrap_or_default();
- // Disable reasoning for providers where post-processing rarely benefits from it.
- // - custom: top-level reasoning_effort (works for local OpenAI-compat servers)
- // - openrouter: nested reasoning object; exclude:true also keeps reasoning text
- // out of the response so it can't pollute structured-output JSON parsing
- let (reasoning_effort, reasoning) = match provider.id.as_str() {
- "custom" => (Some("none".to_string()), None),
- "openrouter" => (
- None,
- Some(crate::llm_client::ReasoningConfig {
- effort: Some("none".to_string()),
- exclude: Some(true),
- }),
- ),
- _ => (None, None),
- };
+ // Ask these providers to skip reasoning/thinking — post-processing rarely
+ // benefits from it and it adds seconds of latency. llm_client picks the
+ // field the endpoint understands and retries without it if rejected.
+ let disable_reasoning = matches!(provider.id.as_str(), "custom" | "openrouter");
if provider.supports_structured_output {
debug!("Using structured outputs for provider '{}'", provider.id);
@@ -254,14 +256,14 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str)
user_content,
Some(system_prompt),
Some(json_schema),
- reasoning_effort.clone(),
- reasoning.clone(),
+ disable_reasoning,
)
.await
{
Ok(Some(content)) => {
// Parse the JSON response to extract the transcription field
- match serde_json::from_str::<serde_json::Value>(&content) {
+ let content = strip_think_block(&content);
+ match serde_json::from_str::<serde_json::Value>(content) {
Ok(json) => {
if let Some(transcription_value) =
json.get(TRANSCRIPTION_FIELD).and_then(|t| t.as_str())
@@ -275,7 +277,7 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str)
return Some(result);
} else {
error!("Structured output response missing 'transcription' field");
- return Some(strip_invisible_chars(&content));
+ return Some(strip_invisible_chars(content));
}
}
Err(e) => {
@@ -283,7 +285,7 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str)
"Failed to parse structured output JSON: {}. Returning raw content.",
e
);
- return Some(strip_invisible_chars(&content));
+ return Some(strip_invisible_chars(content));
}
}
}
@@ -310,13 +312,12 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str)
api_key,
&model,
processed_prompt,
- reasoning_effort,
- reasoning,
+ disable_reasoning,
)
.await
{
Ok(Some(content)) => {
- let content = strip_invisible_chars(&content);
+ let content = strip_invisible_chars(strip_think_block(&content));
debug!(
"LLM post-processing succeeded for provider '{}'. Output length: {} chars",
provider.id,
@@ -927,7 +928,10 @@ pub static ACTION_MAP: Lazy<HashMap<String, Arc<dyn ShortcutAction>>> = Lazy::ne
#[cfg(test)]
mod tests {
- use super::{complete_unless_cancelled, is_blank_transcription, should_use_streaming_overlay};
+ use super::{
+ complete_unless_cancelled, is_blank_transcription, should_use_streaming_overlay,
+ strip_think_block,
+ };
use crate::settings::OverlayStyle;
use std::future;
use std::sync::atomic::{AtomicBool, Ordering};
@@ -976,6 +980,32 @@ mod tests {
assert_eq!(result, None);
}
+ #[test]
+ fn leading_think_block_is_stripped() {
+ assert_eq!(
+ strip_think_block("<think>pondering...</think>Cleaned text."),
+ "Cleaned text."
+ );
+ assert_eq!(
+ strip_think_block(" \n<think>multi\nline</think>\n Cleaned text."),
+ "Cleaned text."
+ );
+ }
+
+ #[test]
+ fn content_without_think_block_is_unchanged() {
+ assert_eq!(strip_think_block("Cleaned text."), "Cleaned text.");
+ assert_eq!(
+ strip_think_block("Mentions <think> mid-sentence."),
+ "Mentions <think> mid-sentence."
+ );
+ // Unclosed block: leave untouched rather than guess
+ assert_eq!(
+ strip_think_block("<think>never closed"),
+ "<think>never closed"
+ );
+ }
+
#[test]
fn live_overlay_uses_streaming_states_only_for_streaming_models() {
assert!(should_use_streaming_overlay(OverlayStyle::Live, true));
diff --git a/src-tauri/src/llm_client.rs b/src-tauri/src/llm_client.rs
index 2a06e173..6ed64c12 100644
--- a/src-tauri/src/llm_client.rs
+++ b/src-tauri/src/llm_client.rs
@@ -1,8 +1,10 @@
use crate::settings::PostProcessProvider;
-use log::debug;
+use log::{debug, info};
use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION, CONTENT_TYPE, REFERER, USER_AGENT};
use serde::{Deserialize, Serialize};
use serde_json::Value;
+use std::collections::HashSet;
+use std::sync::{Mutex, OnceLock};
#[derive(Debug, Serialize)]
struct ChatMessage {
@@ -24,12 +26,86 @@ struct ResponseFormat {
json_schema: JsonSchema,
}
-#[derive(Debug, Serialize, Clone, Default)]
-pub struct ReasoningConfig {
+#[derive(Debug, Serialize, Clone, Default, PartialEq)]
+struct ReasoningConfig {
#[serde(skip_serializing_if = "Option::is_none")]
- pub effort: Option<String>,
+ effort: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
- pub exclude: Option<bool>,
+ exclude: Option<bool>,
+}
+
+/// Request fields used to ask an endpoint to skip reasoning/thinking.
+/// Providers disagree on the field name and accepted values, so at most one of
+/// these is set per request (see `reasoning_disable_params`).
+#[derive(Debug, Serialize, Clone, Default, PartialEq)]
+struct ReasoningParams {
+ #[serde(skip_serializing_if = "Option::is_none")]
+ reasoning_effort: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ reasoning: Option<ReasoningConfig>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ thinking: Option<Value>,
+}
+
+impl ReasoningParams {
+ fn is_empty(&self) -> bool {
+ self.reasoning_effort.is_none() && self.reasoning.is_none() && self.thinking.is_none()
+ }
+}
+
+/// Pick the reasoning-disable request fields an endpoint understands.
+/// Unknown endpoints get the common OpenAI-style field; if they reject it,
+/// the request is retried without it (see `send_chat_completion_with_schema`).
+fn reasoning_disable_params(provider: &PostProcessProvider) -> ReasoningParams {
+ let base_url = provider.base_url.to_lowercase();
+ if base_url.contains("api.deepseek.com") {
+ // DeepSeek rejects reasoning_effort "none" and uses its own field:
+ // https://api-docs.deepseek.com/guides/thinking_mode
+ ReasoningParams {
+ thinking: Some(serde_json::json!({ "type": "disabled" })),
+ ..Default::default()
+ }
+ } else if provider.id == "openrouter" {
+ // OpenRouter nested object; exclude:true also keeps reasoning text out
+ // of the response so it can't pollute structured-output JSON parsing
+ ReasoningParams {
+ reasoning: Some(ReasoningConfig {
+ effort: Some("none".to_string()),
+ exclude: Some(true),
+ }),
+ ..Default::default()
+ }
+ } else {
+ ReasoningParams {
+ reasoning_effort: Some("none".to_string()),
+ ..Default::default()
+ }
+ }
+}
+
+/// Endpoints (base_url|model) that rejected the reasoning-disable fields with a
+/// 4xx. Remembered for the lifetime of the process so every dictation after the
+/// first skips the doomed attempt and goes straight to a plain request.
+fn reasoning_rejections() -> &'static Mutex<HashSet<String>> {
+ static REJECTED: OnceLock<Mutex<HashSet<String>>> = OnceLock::new();
+ REJECTED.get_or_init(|| Mutex::new(HashSet::new()))
+}
+
+fn endpoint_key(provider: &PostProcessProvider, model: &str) -> String {
+ format!("{}|{}", provider.base_url.trim_end_matches('/'), model)
+}
+
+fn is_known_rejected(key: &str) -> bool {
+ reasoning_rejections()
+ .lock()
+ .map(|set| set.contains(key))
+ .unwrap_or(false)
+}
+
+fn remember_rejection(key: String) {
+ if let Ok(mut set) = reasoning_rejections().lock() {
+ set.insert(key);
+ }
}
#[derive(Debug, Serialize)]
@@ -38,10 +114,8 @@ struct ChatCompletionRequest {
messages: Vec<ChatMessage>,
#[serde(skip_serializing_if = "Option::is_none")]
response_format: Option<ResponseFormat>,
- #[serde(skip_serializing_if = "Option::is_none")]
- reasoning_effort: Option<String>,
- #[serde(skip_serializing_if = "Option::is_none")]
- reasoning: Option<ReasoningConfig>,
+ #[serde(flatten)]
+ reasoning: ReasoningParams,
}
#[derive(Debug, Deserialize)]
@@ -113,8 +187,7 @@ pub async fn send_chat_completion(
api_key: String,
model: &str,
prompt: String,
- reasoning_effort: Option<String>,
- reasoning: Option<ReasoningConfig>,
+ disable_reasoning: bool,
) -> Result<Option<String>, String> {
send_chat_completion_with_schema(
provider,
@@ -123,18 +196,21 @@ pub async fn send_chat_completion(
prompt,
None,
None,
- reasoning_effort,
- reasoning,
+ disable_reasoning,
)
.await
}
-/// Send a chat completion request with structured output support
-/// When json_schema is provided, uses structured outputs mode
-/// system_prompt is used as the system message when provided
-/// reasoning_effort sets the OpenAI-style top-level field (e.g., "none", "low", "medium", "high")
-/// reasoning sets the OpenRouter-style nested object (effort + exclude)
-#[allow(clippy::too_many_arguments)]
+/// Send a chat completion request with structured output support.
+/// When json_schema is provided, uses structured outputs mode.
+/// system_prompt is used as the system message when provided.
+///
+/// When disable_reasoning is set, the request carries the reasoning-disable
+/// fields the endpoint is expected to understand. Not every OpenAI-compatible
+/// endpoint accepts them (DeepSeek, Gemini's compat layer, and some OpenRouter
+/// upstreams reject with 400), so a 400/422 answer to such a request triggers
+/// one retry without the fields, and the rejection is remembered per
+/// (base_url, model) so later requests skip the failing attempt entirely.
pub async fn send_chat_completion_with_schema(
provider: &PostProcessProvider,
api_key: String,
@@ -142,8 +218,7 @@ pub async fn send_chat_completion_with_schema(
user_content: String,
system_prompt: Option<String>,
json_schema: Option<Value>,
- reasoning_effort: Option<String>,
- reasoning: Option<ReasoningConfig>,
+ disable_reasoning: bool,
) -> Result<Option<String>, String> {
let base_url = provider.base_url.trim_end_matches('/');
let url = format!("{}/chat/completions", base_url);
@@ -179,22 +254,61 @@ pub async fn send_chat_completion_with_schema(
},
});
- let request_body = ChatCompletionRequest {
+ let key = endpoint_key(provider, model);
+ let reasoning = if disable_reasoning && !is_known_rejected(&key) {
+ reasoning_disable_params(provider)
+ } else {
+ ReasoningParams::default()
+ };
+
+ let mut request_body = ChatCompletionRequest {
model: model.to_string(),
messages,
response_format,
- reasoning_effort,
reasoning,
};
- let response = client
+ let mut response = client
.post(&url)
.json(&request_body)
.send()
.await
.map_err(|e| format!("HTTP request failed: {}", e))?;
+ let mut status = response.status();
+
+ // A 400/422 on a request carrying reasoning-disable fields is almost always
+ // the endpoint rejecting those fields — retry once without them.
+ if !status.is_success()
+ && matches!(status.as_u16(), 400 | 422)
+ && !request_body.reasoning.is_empty()
+ {
+ let error_text = response
+ .text()
+ .await
+ .unwrap_or_else(|_| "Failed to read error response".to_string());
+ info!(
+ "Endpoint rejected request with reasoning disabled (status {}): {}. Retrying without reasoning fields",
+ status, error_text
+ );
+
+ request_body.reasoning = ReasoningParams::default();
+ response = client
+ .post(&url)
+ .json(&request_body)
+ .send()
+ .await
+ .map_err(|e| format!("HTTP request failed: {}", e))?;
+ status = response.status();
+
+ if status.is_success() {
+ info!(
+ "Retry without reasoning fields succeeded; '{}' (model '{}') will skip them from now on",
+ base_url, model
+ );
+ remember_rejection(key);
+ }
+ }
- let status = response.status();
if !status.is_success() {
let error_text = response
.text()
@@ -276,3 +390,96 @@ pub async fn fetch_models(
Ok(models)
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn provider(id: &str, base_url: &str) -> PostProcessProvider {
+ PostProcessProvider {
+ id: id.to_string(),
+ label: id.to_string(),
+ base_url: base_url.to_string(),
+ allow_base_url_edit: true,
+ models_endpoint: None,
+ supports_structured_output: false,
+ }
+ }
+
+ fn request_json(reasoning: ReasoningParams) -> Value {
+ let request = ChatCompletionRequest {
+ model: "test-model".to_string(),
+ messages: vec![ChatMessage {
+ role: "user".to_string(),
+ content: "hi".to_string(),
+ }],
+ response_format: None,
+ reasoning,
+ };
+ serde_json::to_value(&request).unwrap()
+ }
+
+ #[test]
+ fn default_reasoning_params_serialize_to_no_fields() {
+ let json = request_json(ReasoningParams::default());
+ assert!(json.get("reasoning_effort").is_none());
+ assert!(json.get("reasoning").is_none());
+ assert!(json.get("thinking").is_none());
+ }
+
+ #[test]
+ fn custom_provider_uses_top_level_reasoning_effort() {
+ let params = reasoning_disable_params(&provider("custom", "http://localhost:11434/v1"));
+ let json = request_json(params);
+ assert_eq!(json["reasoning_effort"], "none");
+ assert!(json.get("reasoning").is_none());
+ assert!(json.get("thinking").is_none());
+ }
+
+ #[test]
+ fn openrouter_uses_nested_reasoning_object() {
+ let params =
+ reasoning_disable_params(&provider("openrouter", "https://openrouter.ai/api/v1"));
+ let json = request_json(params);
+ assert!(json.get("reasoning_effort").is_none());
+ assert_eq!(json["reasoning"]["effort"], "none");
+ assert_eq!(json["reasoning"]["exclude"], true);
+ assert!(json.get("thinking").is_none());
+ }
+
+ #[test]
+ fn deepseek_base_url_uses_thinking_disabled() {
+ let params = reasoning_disable_params(&provider("custom", "https://api.deepseek.com"));
+ let json = request_json(params);
+ assert!(json.get("reasoning_effort").is_none());
+ assert!(json.get("reasoning").is_none());
+ assert_eq!(json["thinking"]["type"], "disabled");
+ }
+
+ #[test]
+ fn reasoning_params_is_empty_tracks_all_fields() {
+ assert!(ReasoningParams::default().is_empty());
+ assert!(!ReasoningParams {
+ reasoning_effort: Some("none".to_string()),
+ ..Default::default()
+ }
+ .is_empty());
+ assert!(!ReasoningParams {
+ thinking: Some(serde_json::json!({ "type": "disabled" })),
+ ..Default::default()
+ }
+ .is_empty());
+ }
+
+ #[test]
+ fn rejection_memo_is_keyed_by_base_url_and_model() {
+ let deepseek = provider("custom", "https://api.deepseek.com/");
+ let key = endpoint_key(&deepseek, "deepseek-chat");
+ assert_eq!(key, "https://api.deepseek.com|deepseek-chat");
+ assert!(!is_known_rejected(&key));
+ remember_rejection(key.clone());
+ assert!(is_known_rejected(&key));
+ // A different model on the same endpoint is tracked separately
+ assert!(!is_known_rejected(&endpoint_key(&deepseek, "other-model")));
+ }
+}