Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python

Add ENABLE_STREAM_OPTIONS env var for LLM server compatibility

Murtaza Nasir9348ad32a4be
diff --git a/config/env.asr.example b/config/env.asr.example
index 9ab2234c..fd2fd36b 100644
--- a/config/env.asr.example
+++ b/config/env.asr.example
@@ -60,6 +60,12 @@ GPT5_VERBOSITY=medium
# CHAT_GPT5_REASONING_EFFORT=medium
# CHAT_GPT5_VERBOSITY=medium
+# --- LLM Streaming Compatibility ---
+# Some LLM servers (e.g., certain vLLM configurations) don't support OpenAI's
+# stream_options parameter. If chat streaming hangs or fails, try disabling this.
+# Note: When disabled, token usage tracking for chat will not be available.
+# ENABLE_STREAM_OPTIONS=false
+
# --- Transcription Service (ASR Endpoint) ---
# New connector architecture auto-detects ASR mode when ASR_BASE_URL is set.
# USE_ASR_ENDPOINT=true is deprecated but still works for backwards compatibility.
diff --git a/config/env.whisper.example b/config/env.whisper.example
index 63753d62..b2296293 100644
--- a/config/env.whisper.example
+++ b/config/env.whisper.example
@@ -62,6 +62,12 @@ GPT5_VERBOSITY=medium
# CHAT_GPT5_REASONING_EFFORT=medium
# CHAT_GPT5_VERBOSITY=medium
+# --- LLM Streaming Compatibility ---
+# Some LLM servers (e.g., certain vLLM configurations) don't support OpenAI's
+# stream_options parameter. If chat streaming hangs or fails, try disabling this.
+# Note: When disabled, token usage tracking for chat will not be available.
+# ENABLE_STREAM_OPTIONS=false
+
# --- Transcription Service (OpenAI Whisper API) ---
# New connector architecture is enabled by default.
# Available models:
diff --git a/src/services/llm.py b/src/services/llm.py
index e473b38c..c8e9f6bd 100644
--- a/src/services/llm.py
+++ b/src/services/llm.py
@@ -40,6 +40,9 @@ CHAT_MODEL_NAME = os.environ.get("CHAT_MODEL_NAME")
CHAT_GPT5_REASONING_EFFORT = os.environ.get("CHAT_GPT5_REASONING_EFFORT")
CHAT_GPT5_VERBOSITY = os.environ.get("CHAT_GPT5_VERBOSITY")
+# Streaming options - disable for LLM servers that don't support OpenAI's stream_options
+ENABLE_STREAM_OPTIONS = os.environ.get("ENABLE_STREAM_OPTIONS", "true").lower() == "true"
+
def get_chat_config():
"""
@@ -185,7 +188,8 @@ def call_llm_completion(messages, temperature=0.7, response_format=None, stream=
}
# Add stream_options to get usage in final chunk for streaming
- if stream:
+ # Some LLM servers don't support this OpenAI-specific option
+ if stream and ENABLE_STREAM_OPTIONS:
completion_args["stream_options"] = {"include_usage": True}
if using_gpt5:
@@ -308,7 +312,8 @@ def call_chat_completion(messages, temperature=0.7, response_format=None, stream
}
# Add stream_options to get usage in final chunk for streaming
- if stream:
+ # Some LLM servers don't support this OpenAI-specific option
+ if stream and ENABLE_STREAM_OPTIONS:
completion_args["stream_options"] = {"include_usage": True}
if using_gpt5: