Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python

fix: surface finish_reason=length truncation in streaming chat/inquire responses (#349)

Murtaza Nasir15e9ec8af38b
diff --git a/src/services/llm.py b/src/services/llm.py
index 4a3291dd..d665de81 100644
--- a/src/services/llm.py
+++ b/src/services/llm.py
@@ -522,8 +522,17 @@ def process_streaming_with_thinking(stream, user_id=None, operation_type=None, m
content_buffer = ""
in_thinking = False
thinking_buffer = ""
+ finish_reason = None
for chunk in stream:
+ # Capture the provider's final finish_reason so truncation
+ # (finish_reason == 'length') survives to the terminal SSE event
+ # instead of being silently dropped (issue #349).
+ if chunk.choices:
+ chunk_finish = getattr(chunk.choices[0], 'finish_reason', None)
+ if chunk_finish:
+ finish_reason = chunk_finish
+
# Check for usage in final chunk (from stream_options={'include_usage': True})
if hasattr(chunk, 'usage') and chunk.usage and user_id and operation_type:
try:
@@ -613,8 +622,14 @@ def process_streaming_with_thinking(stream, user_id=None, operation_type=None, m
# Regular content
yield f"data: {json.dumps({'delta': content_buffer})}\n\n"
- # Signal the end of the stream
- yield f"data: {json.dumps({'end_of_stream': True})}\n\n"
+ # Signal the end of the stream, propagating the provider's finish_reason
+ # so the frontend can flag truncated (finish_reason == 'length') responses.
+ end_event = {'end_of_stream': True}
+ if finish_reason:
+ end_event['finish_reason'] = finish_reason
+ if finish_reason == 'length':
+ end_event['truncated'] = True
+ yield f"data: {json.dumps(end_event)}\n\n"
diff --git a/static/js/modules/composables/chat.js b/static/js/modules/composables/chat.js
index 47dbe87e..c5d76244 100644
--- a/static/js/modules/composables/chat.js
+++ b/static/js/modules/composables/chat.js
@@ -190,6 +190,13 @@ export function useChat(state, utils) {
}
}
if (data.end_of_stream) {
+ // Provider hit its output-token limit: keep the
+ // partial answer but tell the user it was cut off
+ // instead of presenting it as complete (issue #349).
+ if (data.truncated && assistantMessage && assistantMessage.content) {
+ assistantMessage.content += `\n\n_⚠️ ${t('chat.responseTruncated')}_`;
+ assistantMessage.html = window.renderMarkdownSafe(assistantMessage.content);
+ }
return;
}
if (data.error) {
diff --git a/static/locales/de.json b/static/locales/de.json
index efe876be..fad4181e 100644
--- a/static/locales/de.json
+++ b/static/locales/de.json
@@ -411,6 +411,7 @@
"noMessagesToDownload": "Keine Chat-Nachrichten zum Herunterladen.",
"placeholder": "Eine Frage zu dieser Aufnahme stellen...",
"placeholderWithHint": "Eine Frage zu dieser Aufnahme stellen... (Enter zum Senden, Strg+Enter für neue Zeile)",
+ "responseTruncated": "Antwort wurde vorzeitig beendet, da das Ausgabe-Token-Limit erreicht wurde. Bitten Sie das Modell fortzufahren oder erhöhen Sie CHAT_MAX_TOKENS.",
"send": "Senden",
"suggestedQuestions": "Vorgeschlagene Fragen",
"thinking": "Denke nach...",
@@ -1906,4 +1907,4 @@
"notesWarning": "Es können nur die Notizen einer Aufnahme übernommen werden. Teilnehmer und Tags aller Aufnahmen werden zusammengeführt.",
"notesHint": "Teilnehmer und Tags aller Aufnahmen werden zusammengeführt."
}
-}
\ No newline at end of file
+}
diff --git a/static/locales/en.json b/static/locales/en.json
index 9fe663fb..1083f20d 100644
--- a/static/locales/en.json
+++ b/static/locales/en.json
@@ -411,6 +411,7 @@
"noMessagesToDownload": "No chat messages to download.",
"placeholder": "Ask a question about this recording...",
"placeholderWithHint": "Ask a question about this recording... (Enter to send, Ctrl+Enter for new line)",
+ "responseTruncated": "Response stopped early because the output token limit was reached. Ask the model to continue, or increase CHAT_MAX_TOKENS.",
"send": "Send",
"suggestedQuestions": "Suggested Questions",
"thinking": "Thinking...",
@@ -1906,4 +1907,4 @@
"notesWarning": "Only one recording's notes can be kept. Participants and tags from all recordings are combined.",
"notesHint": "Participants and tags from all recordings are combined."
}
-}
\ No newline at end of file
+}
diff --git a/static/locales/es.json b/static/locales/es.json
index c9c61d32..21c58c86 100644
--- a/static/locales/es.json
+++ b/static/locales/es.json
@@ -411,6 +411,7 @@
"noMessagesToDownload": "No hay mensajes para descargar.",
"placeholder": "Haz una pregunta sobre esta grabación...",
"placeholderWithHint": "Haz una pregunta sobre esta grabación... (Enter para enviar, Ctrl+Enter para nueva línea)",
+ "responseTruncated": "La respuesta se detuvo antes de tiempo porque se alcanzó el límite de tokens de salida. Pida al modelo que continúe o aumente CHAT_MAX_TOKENS.",
"send": "Enviar",
"suggestedQuestions": "Preguntas Sugeridas",
"thinking": "Pensando...",
@@ -1906,4 +1907,4 @@
"notesWarning": "Solo se pueden conservar las notas de una grabación. Los participantes y las etiquetas de todas las grabaciones se combinan.",
"notesHint": "Los participantes y las etiquetas de todas las grabaciones se combinan."
}
-}
\ No newline at end of file
+}
diff --git a/static/locales/fr.json b/static/locales/fr.json
index f1a7103f..4d030177 100644
--- a/static/locales/fr.json
+++ b/static/locales/fr.json
@@ -411,6 +411,7 @@
"noMessagesToDownload": "Aucun message à télécharger.",
"placeholder": "Posez une question sur cet enregistrement...",
"placeholderWithHint": "Posez une question sur cet enregistrement... (Entrée pour envoyer, Ctrl+Entrée pour nouvelle ligne)",
+ "responseTruncated": "La réponse s'est arrêtée prématurément car la limite de jetons de sortie a été atteinte. Demandez au modèle de continuer ou augmentez CHAT_MAX_TOKENS.",
"send": "Envoyer",
"suggestedQuestions": "Questions Suggérées",
"thinking": "Réflexion...",
@@ -1906,4 +1907,4 @@
"notesWarning": "Les notes d'un seul enregistrement peuvent être conservées. Les participants et les étiquettes de tous les enregistrements sont regroupés.",
"notesHint": "Les participants et les étiquettes de tous les enregistrements sont regroupés."
}
-}
\ No newline at end of file
+}
diff --git a/static/locales/pt-BR.json b/static/locales/pt-BR.json
index ffc45553..76cf094d 100644
--- a/static/locales/pt-BR.json
+++ b/static/locales/pt-BR.json
@@ -411,6 +411,7 @@
"noMessagesToDownload": "Nenhuma mensagem de chat para baixar.",
"placeholder": "Faça uma pergunta sobre esta gravação...",
"placeholderWithHint": "Faça uma pergunta sobre esta gravação... (Enter para enviar, Ctrl+Enter para nova linha)",
+ "responseTruncated": "A resposta foi interrompida porque o limite de tokens de saída foi atingido. Peça ao modelo para continuar ou aumente CHAT_MAX_TOKENS.",
"send": "Enviar",
"suggestedQuestions": "Perguntas Sugeridas",
"thinking": "Pensando...",
@@ -1906,4 +1907,4 @@
"notesWarning": "Somente as notas de uma gravação podem ser mantidas. Os participantes e as tags de todas as gravações são combinados.",
"notesHint": "Os participantes e as tags de todas as gravações são combinados."
}
-}
\ No newline at end of file
+}
diff --git a/static/locales/ru.json b/static/locales/ru.json
index 0c8195f3..874bff12 100644
--- a/static/locales/ru.json
+++ b/static/locales/ru.json
@@ -411,6 +411,7 @@
"noMessagesToDownload": "Нет сообщений для скачивания.",
"placeholder": "Задайте вопрос об этой записи...",
"placeholderWithHint": "Задайте вопрос об этой записи... (Enter для отправки, Ctrl+Enter для новой строки)",
+ "responseTruncated": "Ответ был прерван, так как достигнут лимит выходных токенов. Попросите модель продолжить или увеличьте CHAT_MAX_TOKENS.",
"send": "Отправить",
"suggestedQuestions": "Предлагаемые вопросы",
"thinking": "Обдумываю...",
@@ -1906,4 +1907,4 @@
"notesWarning": "Можно сохранить заметки только одной записи. Участники и теги всех записей объединяются.",
"notesHint": "Участники и теги всех записей объединяются."
}
-}
\ No newline at end of file
+}
diff --git a/static/locales/zh.json b/static/locales/zh.json
index 3e1751ca..0cc646bf 100644
--- a/static/locales/zh.json
+++ b/static/locales/zh.json
@@ -411,6 +411,7 @@
"noMessagesToDownload": "没有可下载的聊天消息。",
"placeholder": "询问有关此录音的问题...",
"placeholderWithHint": "询问有关此录音的问题...(按回车发送,Ctrl+回车换行)",
+ "responseTruncated": "回复因达到输出 token 上限而提前中止。可让模型继续回答,或增大 CHAT_MAX_TOKENS。",
"send": "发送",
"suggestedQuestions": "建议问题",
"thinking": "思考中...",
@@ -1906,4 +1907,4 @@
"notesWarning": "只能保留一条录音的备注。所有录音的参与者和标签将被合并。",
"notesHint": "所有录音的参与者和标签将被合并。"
}
-}
\ No newline at end of file
+}
diff --git a/templates/inquire.html b/templates/inquire.html
index 2035db73..84c5f303 100644
--- a/templates/inquire.html
+++ b/templates/inquire.html
@@ -843,6 +843,12 @@
await nextTick();
scrollToBottom();
} else if (data.end_of_stream) {
+ // Flag output-token-limit truncation instead of
+ // presenting the partial answer as complete (issue #349).
+ if (data.truncated && messageAdded && assistantMessage.content) {
+ assistantMessage.content += `\n\n_⚠️ ${t('chat.responseTruncated')}_`;
+ assistantMessage.html = window.renderMarkdownSafe(assistantMessage.content);
+ }
return;
} else if (data.error) {
if (data.budget_exceeded) {
diff --git a/tests/test_cov_llm.py b/tests/test_cov_llm.py
index 8b3cbd68..6c55e06b 100644
--- a/tests/test_cov_llm.py
+++ b/tests/test_cov_llm.py
@@ -71,10 +71,10 @@ def make_completion(content="hello", usage=None, finish_reason="stop", **msg_ext
)
-def make_chunk(content=None, usage=None):
+def make_chunk(content=None, usage=None, finish_reason=None):
"""A streaming chunk with one choice delta."""
delta = SimpleNamespace(content=content)
- choice = SimpleNamespace(delta=delta)
+ choice = SimpleNamespace(delta=delta, finish_reason=finish_reason)
chunk = SimpleNamespace(choices=[choice], usage=usage)
return chunk
@@ -689,6 +689,35 @@ def test_streaming_unclosed_thinking_tag():
assert events[-1] == {"end_of_stream": True}
+def test_streaming_finish_reason_length_marks_truncated():
+ # Provider hit its output-token cap: the terminal event must carry the
+ # truncation signal instead of a bare end_of_stream (issue #349).
+ stream = [make_chunk("partial answer", finish_reason="length")]
+ events = _collect(process_streaming_with_thinking(stream))
+ assert events[-1] == {
+ "end_of_stream": True,
+ "finish_reason": "length",
+ "truncated": True,
+ }
+
+
+def test_streaming_finish_reason_stop_not_truncated():
+ stream = [make_chunk("done", finish_reason="stop")]
+ events = _collect(process_streaming_with_thinking(stream))
+ assert events[-1] == {"end_of_stream": True, "finish_reason": "stop"}
+ assert "truncated" not in events[-1]
+
+
+def test_streaming_chunks_without_finish_reason_attr():
+ # Chunks lacking a finish_reason attribute entirely (older/odd providers)
+ # must still produce a plain end_of_stream event.
+ delta = SimpleNamespace(content="hi")
+ choice = SimpleNamespace(delta=delta)
+ stream = [SimpleNamespace(choices=[choice], usage=None)]
+ events = _collect(process_streaming_with_thinking(stream))
+ assert events[-1] == {"end_of_stream": True}
+
+
def test_streaming_records_usage(monkeypatch):
usage = make_usage(prompt=2, completion=3, total=5)
stream = [make_chunk("hi"), make_chunk(None, usage=usage)]