Speakr is a personal, self-hosted web application designed for transcribing audio recordings
Python
fix: surface finish_reason=length truncation in streaming chat/inquire responses (#349)
Murtaza Nasir15e9ec8af38b
diff --git a/src/services/llm.py b/src/services/llm.pyindex 4a3291dd..d665de81 100644--- a/src/services/llm.py+++ b/src/services/llm.py@@ -522,8 +522,17 @@ def process_streaming_with_thinking(stream, user_id=None, operation_type=None, mcontent_buffer = ""in_thinking = Falsethinking_buffer = ""+ finish_reason = Nonefor chunk in stream:+ # Capture the provider's final finish_reason so truncation+ # (finish_reason == 'length') survives to the terminal SSE event+ # instead of being silently dropped (issue #349).+ if chunk.choices:+ chunk_finish = getattr(chunk.choices[0], 'finish_reason', None)+ if chunk_finish:+ finish_reason = chunk_finish+# Check for usage in final chunk (from stream_options={'include_usage': True})if hasattr(chunk, 'usage') and chunk.usage and user_id and operation_type:try:@@ -613,8 +622,14 @@ def process_streaming_with_thinking(stream, user_id=None, operation_type=None, m# Regular contentyield f"data: {json.dumps({'delta': content_buffer})}\n\n"- # Signal the end of the stream- yield f"data: {json.dumps({'end_of_stream': True})}\n\n"+ # Signal the end of the stream, propagating the provider's finish_reason+ # so the frontend can flag truncated (finish_reason == 'length') responses.+ end_event = {'end_of_stream': True}+ if finish_reason:+ end_event['finish_reason'] = finish_reason+ if finish_reason == 'length':+ end_event['truncated'] = True+ yield f"data: {json.dumps(end_event)}\n\n"diff --git a/static/js/modules/composables/chat.js b/static/js/modules/composables/chat.jsindex 47dbe87e..c5d76244 100644--- a/static/js/modules/composables/chat.js+++ b/static/js/modules/composables/chat.js@@ -190,6 +190,13 @@ export function useChat(state, utils) {}}if (data.end_of_stream) {+ // Provider hit its output-token limit: keep the+ // partial answer but tell the user it was cut off+ // instead of presenting it as complete (issue #349).+ if (data.truncated && assistantMessage && assistantMessage.content) {+ assistantMessage.content += `\n\n_⚠️ ${t('chat.responseTruncated')}_`;+ assistantMessage.html = window.renderMarkdownSafe(assistantMessage.content);+ }return;}if (data.error) {diff --git a/static/locales/de.json b/static/locales/de.jsonindex efe876be..fad4181e 100644--- a/static/locales/de.json+++ b/static/locales/de.json@@ -411,6 +411,7 @@"noMessagesToDownload": "Keine Chat-Nachrichten zum Herunterladen.","placeholder": "Eine Frage zu dieser Aufnahme stellen...","placeholderWithHint": "Eine Frage zu dieser Aufnahme stellen... (Enter zum Senden, Strg+Enter für neue Zeile)",+ "responseTruncated": "Antwort wurde vorzeitig beendet, da das Ausgabe-Token-Limit erreicht wurde. Bitten Sie das Modell fortzufahren oder erhöhen Sie CHAT_MAX_TOKENS.","send": "Senden","suggestedQuestions": "Vorgeschlagene Fragen","thinking": "Denke nach...",@@ -1906,4 +1907,4 @@"notesWarning": "Es können nur die Notizen einer Aufnahme übernommen werden. Teilnehmer und Tags aller Aufnahmen werden zusammengeführt.","notesHint": "Teilnehmer und Tags aller Aufnahmen werden zusammengeführt."}-}\ No newline at end of file+}diff --git a/static/locales/en.json b/static/locales/en.jsonindex 9fe663fb..1083f20d 100644--- a/static/locales/en.json+++ b/static/locales/en.json@@ -411,6 +411,7 @@"noMessagesToDownload": "No chat messages to download.","placeholder": "Ask a question about this recording...","placeholderWithHint": "Ask a question about this recording... (Enter to send, Ctrl+Enter for new line)",+ "responseTruncated": "Response stopped early because the output token limit was reached. Ask the model to continue, or increase CHAT_MAX_TOKENS.","send": "Send","suggestedQuestions": "Suggested Questions","thinking": "Thinking...",@@ -1906,4 +1907,4 @@"notesWarning": "Only one recording's notes can be kept. Participants and tags from all recordings are combined.","notesHint": "Participants and tags from all recordings are combined."}-}\ No newline at end of file+}diff --git a/static/locales/es.json b/static/locales/es.jsonindex c9c61d32..21c58c86 100644--- a/static/locales/es.json+++ b/static/locales/es.json@@ -411,6 +411,7 @@"noMessagesToDownload": "No hay mensajes para descargar.","placeholder": "Haz una pregunta sobre esta grabación...","placeholderWithHint": "Haz una pregunta sobre esta grabación... (Enter para enviar, Ctrl+Enter para nueva línea)",+ "responseTruncated": "La respuesta se detuvo antes de tiempo porque se alcanzó el límite de tokens de salida. Pida al modelo que continúe o aumente CHAT_MAX_TOKENS.","send": "Enviar","suggestedQuestions": "Preguntas Sugeridas","thinking": "Pensando...",@@ -1906,4 +1907,4 @@"notesWarning": "Solo se pueden conservar las notas de una grabación. Los participantes y las etiquetas de todas las grabaciones se combinan.","notesHint": "Los participantes y las etiquetas de todas las grabaciones se combinan."}-}\ No newline at end of file+}diff --git a/static/locales/fr.json b/static/locales/fr.jsonindex f1a7103f..4d030177 100644--- a/static/locales/fr.json+++ b/static/locales/fr.json@@ -411,6 +411,7 @@"noMessagesToDownload": "Aucun message à télécharger.","placeholder": "Posez une question sur cet enregistrement...","placeholderWithHint": "Posez une question sur cet enregistrement... (Entrée pour envoyer, Ctrl+Entrée pour nouvelle ligne)",+ "responseTruncated": "La réponse s'est arrêtée prématurément car la limite de jetons de sortie a été atteinte. Demandez au modèle de continuer ou augmentez CHAT_MAX_TOKENS.","send": "Envoyer","suggestedQuestions": "Questions Suggérées","thinking": "Réflexion...",@@ -1906,4 +1907,4 @@"notesWarning": "Les notes d'un seul enregistrement peuvent être conservées. Les participants et les étiquettes de tous les enregistrements sont regroupés.","notesHint": "Les participants et les étiquettes de tous les enregistrements sont regroupés."}-}\ No newline at end of file+}diff --git a/static/locales/pt-BR.json b/static/locales/pt-BR.jsonindex ffc45553..76cf094d 100644--- a/static/locales/pt-BR.json+++ b/static/locales/pt-BR.json@@ -411,6 +411,7 @@"noMessagesToDownload": "Nenhuma mensagem de chat para baixar.","placeholder": "Faça uma pergunta sobre esta gravação...","placeholderWithHint": "Faça uma pergunta sobre esta gravação... (Enter para enviar, Ctrl+Enter para nova linha)",+ "responseTruncated": "A resposta foi interrompida porque o limite de tokens de saída foi atingido. Peça ao modelo para continuar ou aumente CHAT_MAX_TOKENS.","send": "Enviar","suggestedQuestions": "Perguntas Sugeridas","thinking": "Pensando...",@@ -1906,4 +1907,4 @@"notesWarning": "Somente as notas de uma gravação podem ser mantidas. Os participantes e as tags de todas as gravações são combinados.","notesHint": "Os participantes e as tags de todas as gravações são combinados."}-}\ No newline at end of file+}diff --git a/static/locales/ru.json b/static/locales/ru.jsonindex 0c8195f3..874bff12 100644--- a/static/locales/ru.json+++ b/static/locales/ru.json@@ -411,6 +411,7 @@"noMessagesToDownload": "Нет сообщений для скачивания.","placeholder": "Задайте вопрос об этой записи...","placeholderWithHint": "Задайте вопрос об этой записи... (Enter для отправки, Ctrl+Enter для новой строки)",+ "responseTruncated": "Ответ был прерван, так как достигнут лимит выходных токенов. Попросите модель продолжить или увеличьте CHAT_MAX_TOKENS.","send": "Отправить","suggestedQuestions": "Предлагаемые вопросы","thinking": "Обдумываю...",@@ -1906,4 +1907,4 @@"notesWarning": "Можно сохранить заметки только одной записи. Участники и теги всех записей объединяются.","notesHint": "Участники и теги всех записей объединяются."}-}\ No newline at end of file+}diff --git a/static/locales/zh.json b/static/locales/zh.jsonindex 3e1751ca..0cc646bf 100644--- a/static/locales/zh.json+++ b/static/locales/zh.json@@ -411,6 +411,7 @@"noMessagesToDownload": "没有可下载的聊天消息。","placeholder": "询问有关此录音的问题...","placeholderWithHint": "询问有关此录音的问题...(按回车发送,Ctrl+回车换行)",+ "responseTruncated": "回复因达到输出 token 上限而提前中止。可让模型继续回答,或增大 CHAT_MAX_TOKENS。","send": "发送","suggestedQuestions": "建议问题","thinking": "思考中...",@@ -1906,4 +1907,4 @@"notesWarning": "只能保留一条录音的备注。所有录音的参与者和标签将被合并。","notesHint": "所有录音的参与者和标签将被合并。"}-}\ No newline at end of file+}diff --git a/templates/inquire.html b/templates/inquire.htmlindex 2035db73..84c5f303 100644--- a/templates/inquire.html+++ b/templates/inquire.html@@ -843,6 +843,12 @@await nextTick();scrollToBottom();} else if (data.end_of_stream) {+ // Flag output-token-limit truncation instead of+ // presenting the partial answer as complete (issue #349).+ if (data.truncated && messageAdded && assistantMessage.content) {+ assistantMessage.content += `\n\n_⚠️ ${t('chat.responseTruncated')}_`;+ assistantMessage.html = window.renderMarkdownSafe(assistantMessage.content);+ }return;} else if (data.error) {if (data.budget_exceeded) {diff --git a/tests/test_cov_llm.py b/tests/test_cov_llm.pyindex 8b3cbd68..6c55e06b 100644--- a/tests/test_cov_llm.py+++ b/tests/test_cov_llm.py@@ -71,10 +71,10 @@ def make_completion(content="hello", usage=None, finish_reason="stop", **msg_ext)-def make_chunk(content=None, usage=None):+def make_chunk(content=None, usage=None, finish_reason=None):"""A streaming chunk with one choice delta."""delta = SimpleNamespace(content=content)- choice = SimpleNamespace(delta=delta)+ choice = SimpleNamespace(delta=delta, finish_reason=finish_reason)chunk = SimpleNamespace(choices=[choice], usage=usage)return chunk@@ -689,6 +689,35 @@ def test_streaming_unclosed_thinking_tag():assert events[-1] == {"end_of_stream": True}+def test_streaming_finish_reason_length_marks_truncated():+ # Provider hit its output-token cap: the terminal event must carry the+ # truncation signal instead of a bare end_of_stream (issue #349).+ stream = [make_chunk("partial answer", finish_reason="length")]+ events = _collect(process_streaming_with_thinking(stream))+ assert events[-1] == {+ "end_of_stream": True,+ "finish_reason": "length",+ "truncated": True,+ }+++def test_streaming_finish_reason_stop_not_truncated():+ stream = [make_chunk("done", finish_reason="stop")]+ events = _collect(process_streaming_with_thinking(stream))+ assert events[-1] == {"end_of_stream": True, "finish_reason": "stop"}+ assert "truncated" not in events[-1]+++def test_streaming_chunks_without_finish_reason_attr():+ # Chunks lacking a finish_reason attribute entirely (older/odd providers)+ # must still produce a plain end_of_stream event.+ delta = SimpleNamespace(content="hi")+ choice = SimpleNamespace(delta=delta)+ stream = [SimpleNamespace(choices=[choice], usage=None)]+ events = _collect(process_streaming_with_thinking(stream))+ assert events[-1] == {"end_of_stream": True}++def test_streaming_records_usage(monkeypatch):usage = make_usage(prompt=2, completion=3, total=5)stream = [make_chunk("hi"), make_chunk(None, usage=usage)]