Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python

streaming response

Murtaza Nasirf382ff36e5eb
diff --git a/app.py b/app.py
index fa1b11de..e83a3b2a 100644
--- a/app.py
+++ b/app.py
@@ -1,7 +1,7 @@
# Speakr - Audio Transcription and Summarization App
import os
import sys
-from flask import Flask, render_template, request, jsonify, send_file, redirect, url_for, flash
+from flask import Flask, render_template, request, jsonify, send_file, redirect, url_for, flash, Response
try:
from flask import Markup
except ImportError:
@@ -1046,7 +1046,6 @@ def chat_with_transcription():
return jsonify({'error': 'Chat service is not available (OpenRouter client not configured)'}), 503
# Prepare the system prompt with the transcription
- # output_language_chat = os.environ.get("OUTPUT_LANGUAGE", None) # Get preferred output language for chat - Now from user
user_chat_output_language = current_user.output_language if current_user.is_authenticated else None
language_instruction = ""
@@ -1073,40 +1072,39 @@ Additional context and notes about the meeting:
{recording.notes or "none"}
"""
- # Call the LLM
- try:
- # Prepare messages array with system prompt and conversation history
- messages = [{"role": "system", "content": system_prompt}]
-
- # Add message history if provided
- if message_history:
- messages.extend(message_history)
-
- # Add the current user message
- messages.append({"role": "user", "content": user_message})
-
- completion = client.chat.completions.create(
- model=TEXT_MODEL_NAME,
- messages=messages,
- temperature=0.7,
- max_tokens=int(os.environ.get("CHAT_MAX_TOKENS", "2000"))
- )
-
- response_content = completion.choices[0].message.content
-
- # Convert markdown in the response to HTML
- response_html = md_to_html(response_content)
-
- return jsonify({
- 'response': response_content,
- 'response_html': response_html,
- 'success': True
- })
-
- except Exception as chat_error:
- app.logger.error(f"Error calling OpenRouter API for chat: {str(chat_error)}")
- return jsonify({'error': f'Chat service error: {str(chat_error)}'}), 500
-
+ # Prepare messages array with system prompt and conversation history
+ messages = [{"role": "system", "content": system_prompt}]
+ if message_history:
+ messages.extend(message_history)
+ messages.append({"role": "user", "content": user_message})
+
+ def generate():
+ try:
+ # Enable streaming
+ stream = client.chat.completions.create(
+ model=TEXT_MODEL_NAME,
+ messages=messages,
+ temperature=0.7,
+ max_tokens=int(os.environ.get("CHAT_MAX_TOKENS", "2000")),
+ stream=True
+ )
+
+ for chunk in stream:
+ content = chunk.choices[0].delta.content
+ if content:
+ # Yield the content in SSE format
+ yield f"data: {json.dumps({'delta': content})}\n\n"
+
+ # Signal the end of the stream
+ yield f"data: {json.dumps({'end_of_stream': True})}\n\n"
+
+ except Exception as e:
+ app.logger.error(f"Error during chat stream generation: {str(e)}")
+ # Yield an error message in SSE format
+ yield f"data: {json.dumps({'error': str(e)})}\n\n"
+
+ return Response(generate(), mimetype='text/event-stream')
+
except Exception as e:
app.logger.error(f"Error in chat endpoint: {str(e)}")
return jsonify({'error': str(e)}), 500
diff --git a/static/js/app.js b/static/js/app.js
index 98305b67..7e040c6a 100644
--- a/static/js/app.js
+++ b/static/js/app.js
@@ -1994,36 +1994,29 @@ document.addEventListener('DOMContentLoaded', () => {
if (!chatInput.value.trim() || isChatLoading.value || !selectedRecording.value || selectedRecording.value.status !== 'COMPLETED') {
return;
}
-
+
const message = chatInput.value.trim();
-
- // Ensure chatMessages.value is an array before pushing
+
if (!Array.isArray(chatMessages.value)) {
- console.warn('chatMessages.value was not an array! Resetting. Value was:', chatMessages.value);
- chatMessages.value = []; // Reset if corrupted
+ chatMessages.value = [];
}
-
+
chatMessages.value.push({ role: 'user', content: message });
chatInput.value = '';
isChatLoading.value = true;
-
- // Scroll to bottom of chat
+
await nextTick();
- // Use the correctly named ref
- if (chatMessagesRef.value) {
+ if (chatMessagesRef.value) {
chatMessagesRef.value.scrollTop = chatMessagesRef.value.scrollHeight;
}
-
+
+ let assistantMessage = null;
+
try {
- // Prepare message history for the API call
- // We need to convert our UI messages to the format expected by the API
const messageHistory = chatMessages.value
- .slice(0, -1) // Exclude the message we just added (it will be sent separately)
- .map(msg => ({
- role: msg.role,
- content: msg.content
- }));
-
+ .slice(0, -1) // Exclude the new user message
+ .map(msg => ({ role: msg.role, content: msg.content }));
+
const response = await fetch('/chat', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
@@ -2033,24 +2026,75 @@ document.addEventListener('DOMContentLoaded', () => {
message_history: messageHistory
})
});
-
- const data = await response.json();
- if (!response.ok) throw new Error(data.error || 'Failed to get chat response');
-
- chatMessages.value.push({
- role: 'assistant',
- content: data.response,
- html: data.response_html
- });
+
+ if (!response.ok) {
+ const errorData = await response.json();
+ throw new Error(errorData.error || 'Failed to get chat response');
+ }
+
+ const reader = response.body.getReader();
+ const decoder = new TextDecoder();
+ let buffer = '';
+
+ const processStream = async () => {
+ let isFirstChunk = true;
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+
+ buffer += decoder.decode(value, { stream: true });
+ const lines = buffer.split('\n');
+ buffer = lines.pop();
+
+ for (const line of lines) {
+ if (line.startsWith('data: ')) {
+ const jsonStr = line.substring(6);
+ if (jsonStr) {
+ try {
+ const data = JSON.parse(jsonStr);
+ if (data.delta) {
+ if (isFirstChunk) {
+ isChatLoading.value = false;
+ assistantMessage = reactive({ role: 'assistant', content: '', html: '' });
+ chatMessages.value.push(assistantMessage);
+ isFirstChunk = false;
+ }
+ assistantMessage.content += data.delta;
+ assistantMessage.html = marked.parse(assistantMessage.content);
+
+ await nextTick();
+ if (chatMessagesRef.value) {
+ chatMessagesRef.value.scrollTop = chatMessagesRef.value.scrollHeight;
+ }
+ }
+ if (data.end_of_stream) {
+ return;
+ }
+ if (data.error) {
+ throw new Error(data.error);
+ }
+ } catch (e) {
+ console.error('Error parsing stream data:', e);
+ }
+ }
+ }
+ }
+ }
+ };
+
+ await processStream();
+
} catch (error) {
console.error('Chat Error:', error);
- setGlobalError(`Chat error: ${error.message}`);
- chatMessages.value.push({ role: 'assistant', content: `Error: ${error.message}` });
+ if (assistantMessage) {
+ assistantMessage.content = `Error: ${error.message}`;
+ assistantMessage.html = `<span class="text-red-500">Error: ${error.message}</span>`;
+ } else {
+ chatMessages.value.push({ role: 'assistant', content: `Error: ${error.message}`, html: `<span class="text-red-500">Error: ${error.message}</span>` });
+ }
} finally {
isChatLoading.value = false;
- // Scroll to bottom of chat
await nextTick();
- // Use the correctly named ref
if (chatMessagesRef.value) {
chatMessagesRef.value.scrollTop = chatMessagesRef.value.scrollHeight;
}
diff --git a/templates/index.html b/templates/index.html
index 50139c41..4f2ccafc 100644
--- a/templates/index.html
+++ b/templates/index.html
@@ -11,6 +11,7 @@
<script src="https://unpkg.com/vue@3/dist/vue.global.js"></script>
<link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/all.min.css" rel="stylesheet">
<link href="{{ url_for('static', filename='css/styles.css') }}" rel="stylesheet">
+ <script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
<script src="{{ url_for('static', filename='js/app.js') }}"></script>
</head>
<body class="bg-[var(--bg-primary)] text-[var(--text-primary)]">
@@ -1178,7 +1179,7 @@
class="px-4 py-2 bg-[var(--bg-button)] text-[var(--text-button)] rounded-md hover:bg-[var(--bg-button-hover)] disabled:bg-[var(--bg-tertiary)] disabled:text-[var(--text-muted)] disabled:cursor-not-allowed"
:disabled="!chatInput.trim() || isChatLoading || !selectedRecording || selectedRecording.status !== 'COMPLETED'"
>
- <i class="fas fa-paper-plane"></i>
+ <i class="fas" :class="isChatLoading ? 'fa-spinner fa-spin' : 'fa-paper-plane'"></i>
</button>
</div>
</div>