Speakr is a personal, self-hosted web application designed for transcribing audio recordings
Python
Fix Unicode character rendering in Word document downloads
Murtaza Nasirf9f71bd3e13e
diff --git a/src/app.py b/src/app.pyindex 92be8b53..4e980f93 100644--- a/src/app.py+++ b/src/app.py@@ -957,8 +957,25 @@ def process_markdown_to_docx(doc, content):"""from docx.shared import RGBColor, Ptfrom docx.enum.text import WD_PARAGRAPH_ALIGNMENT+ from docx.oxml.ns import qnimport re+ def ensure_unicode_font(run, text):+ """Ensure the run uses a font that supports the characters in the text."""+ # Check if text contains non-ASCII characters+ try:+ text.encode('ascii')+ # Text is pure ASCII, no special font needed+ except UnicodeEncodeError:+ # Text contains non-ASCII characters, use a font with better Unicode support+ # Use Arial for broad compatibility - it has good Unicode support on most systems+ run.font.name = 'Arial'+ # Set the East Asian font for CJK (Chinese, Japanese, Korean) text+ # This ensures proper rendering in Word+ r = run._element+ r.rPr.rFonts.set(qn('w:eastAsia'), 'Arial')+ return run+def add_formatted_run(paragraph, text):"""Add a run with inline formatting to a paragraph."""if not text:@@ -967,11 +984,11 @@ def process_markdown_to_docx(doc, content):# Pattern for all inline formatting# Order matters: check triple asterisk before double/singlepatterns = [- (r'\*\*\*(.*?)\*\*\*', lambda p, t: setattr(p.add_run(t), 'bold', True) or setattr(p.runs[-1], 'italic', True)), # Bold italic- (r'\*\*(.*?)\*\*', lambda p, t: setattr(p.add_run(t), 'bold', True)), # Bold- (r'(?<!\*)\*(?!\*)(.*?)\*(?!\*)', lambda p, t: setattr(p.add_run(t), 'italic', True)), # Italic with *- (r'\b_(.*?)_\b', lambda p, t: setattr(p.add_run(t), 'italic', True)), # Italic with _- (r'~~(.*?)~~', lambda p, t: setattr(p.add_run(t), 'strike', True)), # Strikethrough+ (r'\*\*\*(.*?)\*\*\*', lambda p, t: (lambda r: (setattr(r, 'bold', True), setattr(r, 'italic', True), ensure_unicode_font(r, t)))(p.add_run(t))), # Bold italic+ (r'\*\*(.*?)\*\*', lambda p, t: (lambda r: (setattr(r, 'bold', True), ensure_unicode_font(r, t)))(p.add_run(t))), # Bold+ (r'(?<!\*)\*(?!\*)(.*?)\*(?!\*)', lambda p, t: (lambda r: (setattr(r, 'italic', True), ensure_unicode_font(r, t)))(p.add_run(t))), # Italic with *+ (r'\b_(.*?)_\b', lambda p, t: (lambda r: (setattr(r, 'italic', True), ensure_unicode_font(r, t)))(p.add_run(t))), # Italic with _+ (r'~~(.*?)~~', lambda p, t: (lambda r: (setattr(r, 'strike', True), ensure_unicode_font(r, t)))(p.add_run(t))), # Strikethrough(r'`([^`]+)`', lambda p, t: add_code_run(p, t)), # Inline code(r'\[([^\]]+)\]\(([^)]+)\)', lambda p, t, u: add_link_run(p, t, u)), # Links]@@ -982,13 +999,22 @@ def process_markdown_to_docx(doc, content):run.font.name = 'Courier New'run.font.size = Pt(10)run.font.color.rgb = RGBColor(220, 20, 60) # Crimson color for code+ # Check if we need Unicode support for code+ try:+ text.encode('ascii')+ except UnicodeEncodeError:+ # Use Consolas as fallback for better Unicode support in monospace+ r = run._element+ r.rPr.rFonts.set(qn('w:eastAsia'), 'Consolas')return rundef add_link_run(para, text, url):"""Add a hyperlink-styled run (note: actual hyperlinks require more complex handling)."""- run = para.add_run(f"{text} ({url})")+ full_text = f"{text} ({url})"+ run = para.add_run(full_text)run.font.color.rgb = RGBColor(0, 0, 255) # Blue color for linksrun.font.underline = True+ ensure_unicode_font(run, full_text)return run# Process the text with all patterns@@ -1009,7 +1035,8 @@ def process_markdown_to_docx(doc, content):if earliest_match:# Add text before the matchif earliest_pos > 0:- paragraph.add_run(remaining_text[:earliest_pos])+ run = paragraph.add_run(remaining_text[:earliest_pos])+ ensure_unicode_font(run, remaining_text[:earliest_pos])# Apply formatting for the matched textif '[' in earliest_match.group(0) and '](' in earliest_match.group(0):@@ -1022,7 +1049,8 @@ def process_markdown_to_docx(doc, content):remaining_text = remaining_text[earliest_match.end():]else:# No more patterns, add the rest as plain text- paragraph.add_run(remaining_text)+ run = paragraph.add_run(remaining_text)+ ensure_unicode_font(run, remaining_text)breakdef parse_table(lines, start_idx):@@ -1084,6 +1112,12 @@ def process_markdown_to_docx(doc, content):run.font.name = 'Courier New'run.font.size = Pt(10)run.font.color.rgb = RGBColor(64, 64, 64)+ # Check if we need Unicode support for code blocks+ try:+ code_text.encode('ascii')+ except UnicodeEncodeError:+ r = run._element+ r.rPr.rFonts.set(qn('w:eastAsia'), 'Consolas')i += 1continue@@ -3786,17 +3820,41 @@ def download_summary_word(recording_id):doc = Document()# Add title- title = doc.add_heading(f'Summary: {recording.title or "Untitled Recording"}', 0)-+ title_text = f'Summary: {recording.title or "Untitled Recording"}'+ title = doc.add_heading(title_text, 0)+ # Check if title needs Unicode font support+ try:+ title_text.encode('ascii')+ except UnicodeEncodeError:+ # Title contains non-ASCII characters+ from docx.oxml.ns import qn+ for run in title.runs:+ run.font.name = 'Arial'+ r = run._element+ r.rPr.rFonts.set(qn('w:eastAsia'), 'Arial')++ # Helper function to add paragraph with Unicode support+ def add_unicode_paragraph(doc, text):+ p = doc.add_paragraph(text)+ try:+ text.encode('ascii')+ except UnicodeEncodeError:+ from docx.oxml.ns import qn+ for run in p.runs:+ run.font.name = 'Arial'+ r = run._element+ r.rPr.rFonts.set(qn('w:eastAsia'), 'Arial')+ return p+# Add metadata- doc.add_paragraph(f'Uploaded: {recording.created_at.strftime("%Y-%m-%d %H:%M")}')+ add_unicode_paragraph(doc, f'Uploaded: {recording.created_at.strftime("%Y-%m-%d %H:%M")}')if recording.meeting_date:- doc.add_paragraph(f'Recording Date: {recording.meeting_date.strftime("%Y-%m-%d")}')+ add_unicode_paragraph(doc, f'Recording Date: {recording.meeting_date.strftime("%Y-%m-%d")}')if recording.participants:- doc.add_paragraph(f'Participants: {recording.participants}')+ add_unicode_paragraph(doc, f'Participants: {recording.participants}')if recording.tags:tags_str = ', '.join([tag.name for tag in recording.tags])- doc.add_paragraph(f'Tags: {tags_str}')+ add_unicode_paragraph(doc, f'Tags: {tags_str}')doc.add_paragraph('') # Empty line# Process markdown content using the helper function@@ -3835,7 +3893,7 @@ def download_summary_word(recording_id):# Use Python's built-in RFC 2231 encoderencoded_value = encode_rfc2231(filename, charset='utf-8')header_value = f'attachment; filename*={encoded_value}'- app.logger.error(f"DEBUG CHINESE FILENAME (RFC2231): Original='{filename}', Header='{header_value}'")+ app.logger.info(f"DEBUG CHINESE FILENAME (RFC2231): Original='{filename}', Header='{header_value}'")response.headers['Content-Disposition'] = header_valueexcept Exception as e:# Fallback to simple attachment with generic name@@ -3878,11 +3936,34 @@ def download_chat_word(recording_id):doc = Document()# Add title- title = doc.add_heading(f'Chat Conversation: {recording.title or "Untitled Recording"}', 0)-+ title_text = f'Chat Conversation: {recording.title or "Untitled Recording"}'+ title = doc.add_heading(title_text, 0)+ # Check if title needs Unicode font support+ try:+ title_text.encode('ascii')+ except UnicodeEncodeError:+ from docx.oxml.ns import qn+ for run in title.runs:+ run.font.name = 'Arial'+ r = run._element+ r.rPr.rFonts.set(qn('w:eastAsia'), 'Arial')++ # Helper function to add paragraph with Unicode support+ def add_unicode_paragraph(doc, text):+ p = doc.add_paragraph(text)+ try:+ text.encode('ascii')+ except UnicodeEncodeError:+ from docx.oxml.ns import qn+ for run in p.runs:+ run.font.name = 'Arial'+ r = run._element+ r.rPr.rFonts.set(qn('w:eastAsia'), 'Arial')+ return p+# Add metadata- doc.add_paragraph(f'Recording Date: {recording.created_at.strftime("%Y-%m-%d %H:%M")}')- doc.add_paragraph(f'Chat Export Date: {datetime.utcnow().strftime("%Y-%m-%d %H:%M")}')+ add_unicode_paragraph(doc, f'Recording Date: {recording.created_at.strftime("%Y-%m-%d %H:%M")}')+ add_unicode_paragraph(doc, f'Chat Export Date: {datetime.utcnow().strftime("%Y-%m-%d %H:%M")}')doc.add_paragraph('') # Empty line# Add chat messages@@ -3894,13 +3975,16 @@ def download_chat_word(recording_id):# Add role headerif role == 'user':p = doc.add_paragraph()- p.add_run('You: ').bold = True+ run = p.add_run('You: ')+ run.bold = Trueelif role == 'assistant':p = doc.add_paragraph()- p.add_run('Assistant: ').bold = True+ run = p.add_run('Assistant: ')+ run.bold = Trueelse:p = doc.add_paragraph()- p.add_run(f'{role.title()}: ').bold = True+ run = p.add_run(f'{role.title()}: ')+ run.bold = True# Add thinking content if presentif thinking and role == 'assistant':@@ -3948,7 +4032,7 @@ def download_chat_word(recording_id):# Use Python's built-in RFC 2231 encoderencoded_value = encode_rfc2231(filename, charset='utf-8')header_value = f'attachment; filename*={encoded_value}'- app.logger.error(f"DEBUG CHINESE FILENAME (RFC2231): Original='{filename}', Header='{header_value}'")+ app.logger.info(f"DEBUG CHINESE FILENAME (RFC2231): Original='{filename}', Header='{header_value}'")response.headers['Content-Disposition'] = header_valueexcept Exception as e:# Fallback to simple attachment with generic name@@ -4176,17 +4260,40 @@ def download_notes_word(recording_id):doc = Document()# Add title- title = doc.add_heading(f'Notes: {recording.title or "Untitled Recording"}', 0)-+ title_text = f'Notes: {recording.title or "Untitled Recording"}'+ title = doc.add_heading(title_text, 0)+ # Check if title needs Unicode font support+ try:+ title_text.encode('ascii')+ except UnicodeEncodeError:+ from docx.oxml.ns import qn+ for run in title.runs:+ run.font.name = 'Arial'+ r = run._element+ r.rPr.rFonts.set(qn('w:eastAsia'), 'Arial')++ # Helper function to add paragraph with Unicode support+ def add_unicode_paragraph(doc, text):+ p = doc.add_paragraph(text)+ try:+ text.encode('ascii')+ except UnicodeEncodeError:+ from docx.oxml.ns import qn+ for run in p.runs:+ run.font.name = 'Arial'+ r = run._element+ r.rPr.rFonts.set(qn('w:eastAsia'), 'Arial')+ return p+# Add metadata- doc.add_paragraph(f'Uploaded: {recording.created_at.strftime("%Y-%m-%d %H:%M")}')+ add_unicode_paragraph(doc, f'Uploaded: {recording.created_at.strftime("%Y-%m-%d %H:%M")}')if recording.meeting_date:- doc.add_paragraph(f'Recording Date: {recording.meeting_date.strftime("%Y-%m-%d")}')+ add_unicode_paragraph(doc, f'Recording Date: {recording.meeting_date.strftime("%Y-%m-%d")}')if recording.participants:- doc.add_paragraph(f'Participants: {recording.participants}')+ add_unicode_paragraph(doc, f'Participants: {recording.participants}')if recording.tags:tags_str = ', '.join([tag.name for tag in recording.tags])- doc.add_paragraph(f'Tags: {tags_str}')+ add_unicode_paragraph(doc, f'Tags: {tags_str}')doc.add_paragraph('') # Empty line# Process markdown content using the helper function@@ -4225,7 +4332,7 @@ def download_notes_word(recording_id):# Use Python's built-in RFC 2231 encoderencoded_value = encode_rfc2231(filename, charset='utf-8')header_value = f'attachment; filename*={encoded_value}'- app.logger.error(f"DEBUG CHINESE FILENAME (RFC2231): Original='{filename}', Header='{header_value}'")+ app.logger.info(f"DEBUG CHINESE FILENAME (RFC2231): Original='{filename}', Header='{header_value}'")response.headers['Content-Disposition'] = header_valueexcept Exception as e:# Fallback to simple attachment with generic namediff --git a/static/js/app.js b/static/js/app.jsindex 6e42358d..a53e0b3c 100644--- a/static/js/app.js+++ b/static/js/app.js@@ -2217,6 +2217,11 @@ document.addEventListener('DOMContentLoaded', async () => {}};+ const startStatusPolling = (fileItem, recordingId) => {+ fileItem.recordingId = recordingId;+ pollProcessingStatus(fileItem);+ };+const pollProcessingStatus = (fileItem) => {if (pollInterval.value) clearInterval(pollInterval.value);diff --git a/templates/account.html b/templates/account.htmlindex 30cd23ea..dc01f5e2 100644--- a/templates/account.html+++ b/templates/account.html@@ -156,6 +156,9 @@// Function to apply the theme based on localStoragefunction applyTheme() {+ // Guard against early execution+ if (!document.documentElement) return;+// Apply dark modeconst savedMode = localStorage.getItem('darkMode');const prefersDark = window.matchMedia('(prefers-color-scheme: dark)').matches;diff --git a/templates/admin.html b/templates/admin.htmlindex ccb2d5e9..5e369279 100644--- a/templates/admin.html+++ b/templates/admin.html@@ -64,6 +64,9 @@// Function to apply the theme based on localStoragefunction applyTheme() {+ // Guard against early execution+ if (!document.documentElement) return;+// Apply dark modeconst savedMode = localStorage.getItem('darkMode');const prefersDark = window.matchMedia('(prefers-color-scheme: dark)').matches;diff --git a/templates/login.html b/templates/login.htmlindex 6db6766b..0df3560d 100644--- a/templates/login.html+++ b/templates/login.html@@ -18,6 +18,9 @@<script>// Function to apply the theme based on localStoragefunction applyTheme() {+ // Guard against early execution+ if (!document.documentElement) return;+// Apply dark modeconst savedMode = localStorage.getItem('darkMode');const prefersDark = window.matchMedia('(prefers-color-scheme: dark)').matches;diff --git a/templates/register.html b/templates/register.htmlindex 92397b88..fd8fad3b 100644--- a/templates/register.html+++ b/templates/register.html@@ -18,6 +18,9 @@<script>// Function to apply the theme based on localStoragefunction applyTheme() {+ // Guard against early execution+ if (!document.documentElement) return;+// Apply dark modeconst savedMode = localStorage.getItem('darkMode');const prefersDark = window.matchMedia('(prefers-color-scheme: dark)').matches;diff --git a/templates/share.html b/templates/share.htmlindex 3f95d544..72cf3163 100644--- a/templates/share.html+++ b/templates/share.html@@ -38,6 +38,9 @@// Function to apply the theme based on localStorage and system preferencefunction applyTheme() {+ // Guard against early execution+ if (!document.documentElement) return;+// Apply dark modeconst savedMode = localStorage.getItem('darkMode');const prefersDark = window.matchMedia('(prefers-color-scheme: dark)').matches;diff --git a/test_unicode_docx.py b/test_unicode_docx.pynew file mode 100644index 00000000..b0f47909--- /dev/null+++ b/test_unicode_docx.py@@ -0,0 +1,75 @@+#!/usr/bin/env python+"""Test script to verify Unicode support in Word documents."""++from docx import Document+from docx.shared import RGBColor, Pt+from docx.oxml.ns import qn+import sys++def test_unicode_support():+ """Test creating a Word document with various Unicode characters."""+ doc = Document()++ # Test data with various Unicode characters+ test_strings = [+ ("Chinese (Simplified)", "第二次研究 - 美食与饮品词汇学习"),+ ("Chinese (Traditional)", "第二次研究 - 美食與飲品詞彙學習"),+ ("Japanese", "これはテストです。日本語のテキスト。"),+ ("Korean", "이것은 테스트입니다. 한국어 텍스트."),+ ("Arabic", "هذا اختبار. نص عربي."),+ ("Hebrew", "זה מבחן. טקסט עברי."),+ ("Russian", "Это тест. Русский текст."),+ ("Greek", "Αυτό είναι ένα τεστ. Ελληνικό κείμενο."),+ ("Thai", "นี่คือการทดสอบ ข้อความภาษาไทย"),+ ("Emoji", "Testing emojis: 😀 🎉 🚀 ❤️"),+ ("Mixed", "Summary: 第二次研究 with English and 中文"),+ ]++ # Add title+ title_text = "Unicode Font Support Test Document"+ title = doc.add_heading(title_text, 0)++ # Add test strings+ for language, text in test_strings:+ # Add language label+ p_label = doc.add_paragraph()+ run = p_label.add_run(f"{language}: ")+ run.bold = True++ # Add test text+ p_text = doc.add_paragraph(text)++ # Apply Unicode font support if needed+ try:+ text.encode('ascii')+ print(f"✓ {language}: ASCII only, no special font needed")+ except UnicodeEncodeError:+ print(f"✓ {language}: Non-ASCII detected, applying Arial font")+ for run in p_text.runs:+ run.font.name = 'Arial'+ r = run._element+ r.rPr.rFonts.set(qn('w:eastAsia'), 'Arial')++ # Save document+ output_file = "test_unicode_output.docx"+ doc.save(output_file)+ print(f"\nTest document saved as: {output_file}")+ print("Please open this document in Microsoft Word to verify all characters display correctly.")++ return True++if __name__ == "__main__":+ try:+ # Check if python-docx is installed+ import docx+ print("python-docx is installed. Version:", docx.__version__)+ except ImportError:+ print("ERROR: python-docx is not installed. Please install it with: pip install python-docx")+ sys.exit(1)++ print("Testing Unicode support in Word documents...\n")+ if test_unicode_support():+ print("\nTest completed successfully!")+ print("\nNOTE: The fix implemented uses Arial font which has broad Unicode support.")+ print("The solution automatically detects non-ASCII characters and applies appropriate font settings.")+ print("This ensures compatibility across different languages without forcing a specific font on all users.")\ No newline at end of file