Speakr is a personal, self-hosted web application designed for transcribing audio recordings
Python
Add parallel uploads with concurrency limit and SHA-256 duplicate detection
Murtaza Nasir9134a9e1e7f9
diff --git a/src/api/recordings.py b/src/api/recordings.pyindex c3c301e1..574ce457 100644--- a/src/api/recordings.py+++ b/src/api/recordings.py@@ -37,6 +37,7 @@ from src.services.embeddings import process_recording_chunksfrom src.file_exporter import export_recording, mark_export_as_deletedfrom src.utils.ffprobe import get_codec_info, get_creation_date, FFProbeErrorfrom src.utils.audio_conversion import convert_if_needed+from src.utils.file_hash import compute_file_sha256# Create blueprintrecordings_bp = Blueprint('recordings', __name__)@@ -1978,6 +1979,27 @@ def upload_file():# Get final file size (of original or converted file)final_file_size = os.path.getsize(filepath)+ # Compute file hash for duplicate detection (hash the final/converted file)+ file_hash = None+ duplicate_warning = None+ try:+ file_hash = compute_file_sha256(filepath)+ existing = Recording.query.filter_by(+ user_id=current_user.id, file_hash=file_hash+ ).first()+ if existing:+ duplicate_warning = {+ 'existing_recording_id': existing.id,+ 'existing_title': existing.title,+ 'existing_created_at': existing.created_at.isoformat() if existing.created_at else None+ }+ current_app.logger.info(+ f"Duplicate file detected for user {current_user.id}: "+ f"hash={file_hash[:12]}... matches recording {existing.id}"+ )+ except Exception as e:+ current_app.logger.warning(f"Could not compute file hash: {e}")+# Determine MIME type of the final filemime_type, _ = mimetypes.guess_type(filepath)current_app.logger.info(f"Final MIME type: {mime_type} for file {filepath}")@@ -2119,7 +2141,8 @@ def upload_file():mime_type=mime_type,notes=notes,folder_id=selected_folder.id if selected_folder else None,- processing_source='upload' # Track that this was manually uploaded+ processing_source='upload', # Track that this was manually uploaded+ file_hash=file_hash)db.session.add(recording)db.session.commit()@@ -2160,7 +2183,10 @@ def upload_file():)current_app.logger.info(f"Transcription job queued for recording ID: {recording.id}")- return jsonify(recording.to_dict(viewer_user=current_user)), 202+ response_data = recording.to_dict(viewer_user=current_user)+ if duplicate_warning:+ response_data['duplicate_warning'] = duplicate_warning+ return jsonify(response_data), 202except RequestEntityTooLarge:max_size_mb = current_app.config['MAX_CONTENT_LENGTH'] / (1024 * 1024)diff --git a/src/file_monitor.py b/src/file_monitor.pyindex 10f7b6ba..f5d5f86c 100644--- a/src/file_monitor.py+++ b/src/file_monitor.py@@ -361,6 +361,14 @@ class FileMonitor:self.logger.error(f"FFmpeg conversion failed: {e}")raise+ # Compute file hash for duplicate detection+ file_hash = None+ try:+ from src.utils.file_hash import compute_file_sha256+ file_hash = compute_file_sha256(str(final_path))+ except Exception as e:+ self.logger.warning(f"Could not compute file hash: {e}")+# Get file size and MIME typefile_size = final_path.stat().st_sizemime_type, _ = mimetypes.guess_type(str(final_path))@@ -376,6 +384,16 @@ class FileMonitor:meeting_date = nowself.logger.debug("No metadata creation date found, using current time")+ # Check for duplicate+ if file_hash:+ existing = Recording.query.filter_by(user_id=user_id, file_hash=file_hash).first()+ if existing:+ self.logger.warning(+ f"Duplicate file detected for user {user_id}: "+ f"hash={file_hash[:12]}... matches recording {existing.id} "+ f"({existing.title}). Processing anyway."+ )+recording = Recording(audio_path=str(final_path),original_filename=original_filename,@@ -386,7 +404,8 @@ class FileMonitor:user_id=user_id,mime_type=mime_type,is_inbox=True, # Auto-processed files go to inbox- processing_source='auto_process' # Track that this was auto-processed+ processing_source='auto_process', # Track that this was auto-processed+ file_hash=file_hash)db.session.add(recording)diff --git a/src/init_db.py b/src/init_db.pyindex 59f448cf..01a6867a 100644--- a/src/init_db.py+++ b/src/init_db.py@@ -517,6 +517,15 @@ def initialize_database(app):except Exception as e:app.logger.warning(f"Could not create unique index on user.sso_subject: {e}")+ # Add file_hash column for duplicate detection+ if add_column_if_not_exists(engine, 'recording', 'file_hash', 'VARCHAR(64)'):+ app.logger.info("Added file_hash column to recording table")+ try:+ if create_index_if_not_exists(engine, 'ix_recording_user_file_hash', 'recording', 'user_id, file_hash'):+ app.logger.info("Created index ix_recording_user_file_hash on recording (user_id, file_hash)")+ except Exception as e:+ app.logger.warning(f"Could not create index on recording (user_id, file_hash): {e}")+# Add folder_id column to recording table for folders featureif add_column_if_not_exists(engine, 'recording', 'folder_id', 'INTEGER'):app.logger.info("Added folder_id column to recording table")diff --git a/src/models/recording.py b/src/models/recording.pyindex 225b99e5..8ddb5a2f 100644--- a/src/models/recording.py+++ b/src/models/recording.py@@ -41,6 +41,7 @@ class Recording(db.Model):summarization_duration_seconds = db.Column(db.Integer, nullable=True) # Time taken for summarizationprocessing_source = db.Column(db.String(50), default='upload') # upload, auto_process, recordingerror_message = db.Column(db.Text, nullable=True) # Store detailed error messages+ file_hash = db.Column(db.String(64), nullable=True) # SHA-256 hash for duplicate detection# Auto-deletion and archival fieldsaudio_deleted_at = db.Column(db.DateTime, nullable=True) # When audio file was deleted (null = not deleted)diff --git a/src/utils/file_hash.py b/src/utils/file_hash.pynew file mode 100644index 00000000..7dab3860--- /dev/null+++ b/src/utils/file_hash.py@@ -0,0 +1,24 @@+"""File hashing utility for duplicate detection."""++import hashlib+++def compute_file_sha256(filepath, chunk_size=8192):+ """+ Compute SHA-256 hash of a file, reading in chunks to handle large files.++ Args:+ filepath: Path to the file to hash+ chunk_size: Size of chunks to read at a time (default 8KB)++ Returns:+ 64-character hex digest string+ """+ sha256 = hashlib.sha256()+ with open(filepath, 'rb') as f:+ while True:+ chunk = f.read(chunk_size)+ if not chunk:+ break+ sha256.update(chunk)+ return sha256.hexdigest()diff --git a/static/js/app.modular.js b/static/js/app.modular.jsindex 76f22be6..60aa63e6 100644--- a/static/js/app.modular.js+++ b/static/js/app.modular.js@@ -1634,6 +1634,63 @@ document.addEventListener('DOMContentLoaded', async () => {}});+ // Track completed recording IDs to detect new completions+ const completedRecordingIds = new Set();++ // Watch allJobs for completed/failed transitions - update local recordings state+ watch(allJobs, async (jobs) => {+ for (const job of jobs) {+ if (job.job_status === 'completed' && !completedRecordingIds.has(job.recording_id)) {+ completedRecordingIds.add(job.recording_id);+ try {+ const fullResponse = await fetch(`/api/recordings/${job.recording_id}`);+ if (fullResponse.ok) {+ const data = await fullResponse.json();+ const idx = recordings.value.findIndex(r => r.id === job.recording_id);+ if (idx !== -1) {+ recordings.value[idx] = data;+ }+ if (selectedRecording.value?.id === job.recording_id) {+ selectedRecording.value = data;+ }+ // Update display name on upload queue item+ const queueItem = uploadQueue.value.find(u => u.recordingId === job.recording_id);+ if (queueItem) {+ queueItem.displayName = data.title || data.original_filename || queueItem.file?.name;+ queueItem.status = 'completed';+ }+ // Refresh token budget+ if (typeof loadTokenBudget === 'function') loadTokenBudget();+ }+ } catch (err) {+ console.error(`Error fetching completed recording ${job.recording_id}:`, err);+ }+ } else if (job.job_status === 'failed' && !completedRecordingIds.has(`fail_${job.recording_id}`)) {+ completedRecordingIds.add(`fail_${job.recording_id}`);+ try {+ const failedResponse = await fetch(`/api/recordings/${job.recording_id}`);+ if (failedResponse.ok) {+ const failedData = await failedResponse.json();+ const idx = recordings.value.findIndex(r => r.id === job.recording_id);+ if (idx !== -1) {+ recordings.value[idx] = failedData;+ }+ if (selectedRecording.value?.id === job.recording_id) {+ selectedRecording.value = failedData;+ }+ const queueItem = uploadQueue.value.find(u => u.recordingId === job.recording_id);+ if (queueItem) {+ queueItem.status = 'failed';+ queueItem.error = failedData.error_message || 'Processing failed on server.';+ }+ }+ } catch (err) {+ console.error(`Error fetching failed recording ${job.recording_id}:`, err);+ }+ }+ }+ }, { deep: true });+// Get job details for a recordingconst getJobDetails = (recordingId) => {return jobQueueDetails.value[recordingId] || null;@@ -1709,9 +1766,6 @@ document.addEventListener('DOMContentLoaded', async () => {const unifiedProgressItems = computed(() => {const items = new Map(); // Key by recordingId or clientId- // Get the currently uploading file's clientId for special handling- const currentUploadClientId = currentlyProcessingFile.value?.clientId;-// 1. First, add all backend jobs (these have the most accurate status)for (const job of allJobs.value) {const key = `rec_${job.recording_id}`;@@ -1752,80 +1806,50 @@ document.addEventListener('DOMContentLoaded', async () => {// 2. Add upload queue items (client-side tracking)for (const upload of uploadQueue.value) {- // Check if this is the currently uploading file- const isCurrentUpload = upload.clientId === currentUploadClientId;-- // If we have a recordingId and it's already tracked from jobs, check if we should merge+ // If we have a recordingId and it's already tracked from jobs, merge upload infoif (upload.recordingId) {const key = `rec_${upload.recordingId}`;const existing = items.get(key);if (existing) {- // Merge upload info into existing itemexisting.clientId = upload.clientId;existing.file = upload.file;- // If this is the current upload, use the global progress refs- if (isCurrentUpload && ['uploading', 'pending', 'processing', 'summarizing'].includes(upload.status)) {- // Map upload status to unified status- if (upload.status === 'uploading') {- existing.status = 'uploading';- } else if (upload.status === 'summarizing') {- existing.status = 'summarizing';- } else if (upload.status === 'processing' || upload.status === 'pending') {- existing.status = 'transcribing';- }- existing.progress = processingProgress.value;- existing.progressMessage = processingMessage.value || 'Processing...';+ existing.duplicateWarning = upload.duplicateWarning || null;+ // If still uploading, override job status with upload status+ if (upload.status === 'uploading') {+ existing.status = 'uploading';+ existing.progress = upload.progress || 0;+ existing.progressMessage = 'Uploading...';existing.title = upload.displayName || upload.file?.name || existing.title;}continue;}}- // Determine unified status from upload status+ // Determine unified status from upload status (per-item progress)let unifiedStatus = 'ready';- let progressVal = 0;+ let progressVal = upload.progress || 0;let progressMsg = 'Waiting to upload...';- // If this is the currently processing file, use global progress refs- if (isCurrentUpload) {- progressVal = processingProgress.value;- progressMsg = processingMessage.value || 'Processing...';-- if (upload.status === 'uploading') {- unifiedStatus = 'uploading';- } else if (upload.status === 'pending' || upload.status === 'processing' || upload.status === 'PROCESSING') {- unifiedStatus = 'transcribing';- } else if (upload.status === 'summarizing' || upload.status === 'SUMMARIZING') {- unifiedStatus = 'summarizing';- } else if (upload.status === 'completed' || upload.status === 'COMPLETED') {- unifiedStatus = 'completed';- progressMsg = 'Done';- } else if (upload.status === 'failed' || upload.status === 'FAILED') {- unifiedStatus = 'failed';- progressMsg = upload.error || 'Processing failed';- } else if (upload.status === 'ready') {- unifiedStatus = 'ready';- progressMsg = 'Waiting to upload...';- }- } else {- // Not the current upload - use item's own status- if (upload.status === 'uploading') {- unifiedStatus = 'uploading';- progressMsg = 'Uploading...';- } else if (upload.status === 'completed' || upload.status === 'COMPLETED') {- unifiedStatus = 'completed';- progressMsg = 'Done';- } else if (upload.status === 'failed' || upload.status === 'FAILED') {- unifiedStatus = 'upload_failed';- progressMsg = upload.error || 'Upload failed';- } else if (upload.status === 'ready') {- unifiedStatus = 'ready';- progressMsg = 'Waiting to upload...';- } else if (upload.status === 'queued') {- unifiedStatus = 'ready';- progressMsg = 'Waiting to upload...';- }+ if (upload.status === 'uploading') {+ unifiedStatus = 'uploading';+ progressMsg = 'Uploading...';+ } else if (upload.status === 'pending') {+ unifiedStatus = 'queued';+ progressVal = 100;+ progressMsg = 'Uploaded, waiting for processing...';+ } else if (upload.status === 'completed' || upload.status === 'COMPLETED') {+ unifiedStatus = 'completed';+ progressMsg = 'Done';+ } else if (upload.status === 'failed' || upload.status === 'FAILED') {+ unifiedStatus = 'upload_failed';+ progressMsg = upload.error || 'Upload failed';+ } else if (upload.status === 'ready') {+ unifiedStatus = 'ready';+ progressMsg = 'Waiting to upload...';+ } else if (upload.status === 'queued') {+ unifiedStatus = 'ready';+ progressMsg = 'Waiting to upload...';}const key = upload.recordingId ? `rec_${upload.recordingId}` : `client_${upload.clientId}`;@@ -1846,6 +1870,7 @@ document.addEventListener('DOMContentLoaded', async () => {progressMessage: progressMsg,queuePosition: null,errorMessage: upload.status === 'failed' ? upload.error : null,+ duplicateWarning: upload.duplicateWarning || null,file: upload.file,source: 'upload'});diff --git a/static/js/modules/composables/upload.js b/static/js/modules/composables/upload.jsindex 7b033c0c..39f50f36 100644--- a/static/js/modules/composables/upload.js+++ b/static/js/modules/composables/upload.js@@ -268,6 +268,39 @@ export function useUpload(state, utils) {}};+ // --- Parallel Upload System ---+ // Concurrency limiter: max 3 simultaneous uploads+ const MAX_CONCURRENT_UPLOADS = 3;+ let activeUploadCount = 0;+ const pendingUploadQueue = []; // Functions waiting for a slot++ const acquireUploadSlot = () => {+ return new Promise(resolve => {+ if (activeUploadCount < MAX_CONCURRENT_UPLOADS) {+ activeUploadCount++;+ resolve();+ } else {+ pendingUploadQueue.push(resolve);+ }+ });+ };++ const releaseUploadSlot = () => {+ activeUploadCount--;+ if (pendingUploadQueue.length > 0) {+ activeUploadCount++;+ const next = pendingUploadQueue.shift();+ next();+ }+ // When all uploads are done, clear processing active flag+ const stillUploading = uploadQueue.value.some(item =>+ ['uploading', 'ready'].includes(item.status)+ );+ if (!stillUploading) {+ isProcessingActive.value = false;+ }+ };+const resetCurrentFileProcessingState = () => {if (pollInterval.value) clearInterval(pollInterval.value);pollInterval.value = null;@@ -276,424 +309,209 @@ export function useUpload(state, utils) {processingMessage.value = '';};- const startProcessingQueue = async () => {- console.log("Attempting to start processing queue...");- if (isProcessingActive.value) {- console.log("Queue processor already active.");- return;- }+ /**+ * Upload a single file to the server.+ * Acquires a concurrency slot, uploads, then releases.+ * Status updates are per-item (no global processingProgress).+ */+ const uploadSingleFile = async (fileItem) => {+ await acquireUploadSlot();- isProcessingActive.value = true;- resetCurrentFileProcessingState();+ fileItem.status = 'uploading';+ fileItem.progress = 5;- const nextFileItem = uploadQueue.value.find(item => item.status === 'ready' || item.status === 'queued');+ try {+ const formData = new FormData();+ formData.append('file', fileItem.file);- if (nextFileItem) {- console.log(`Processing next file: ${nextFileItem.file.name} (Client ID: ${nextFileItem.clientId})`);- currentlyProcessingFile.value = nextFileItem;+ // Send file's lastModified timestamp for meeting_date+ if (fileItem.file.lastModified) {+ const lastModified = fileItem.file.lastModified;+ formData.append('file_last_modified', lastModified.toString());+ }- // Check if this is a "reload" item (existing recording being tracked)- if (nextFileItem.clientId.startsWith('reload-')) {- console.log(`Skipping upload for existing recording: ${nextFileItem.recordingId}`);- nextFileItem.status = 'processing';- startStatusPolling(nextFileItem, nextFileItem.recordingId);- return;+ if (fileItem.notes) {+ formData.append('notes', fileItem.notes);}- nextFileItem.status = 'uploading';- processingMessage.value = 'Preparing upload...';- processingProgress.value = 5;+ // Add tags if selected+ const tagsToUse = fileItem.tags || selectedTags.value || [];+ tagsToUse.forEach((tag, index) => {+ const tagId = tag.id || tag;+ formData.append(`tag_ids[${index}]`, tagId);+ });- try {- const formData = new FormData();- formData.append('file', nextFileItem.file);-- // Send file's lastModified timestamp for meeting_date- if (nextFileItem.file.lastModified) {- const lastModified = nextFileItem.file.lastModified;- formData.append('file_last_modified', lastModified.toString());- console.log(`[Upload] File lastModified: ${lastModified} (${new Date(lastModified).toISOString()})`);- }+ // Add folder if selected+ const folderToUse = fileItem.folder_id || selectedFolderId.value;+ if (folderToUse) {+ formData.append('folder_id', folderToUse);+ }- if (nextFileItem.notes) {- formData.append('notes', nextFileItem.notes);- }+ // Add ASR options+ const asrOpts = fileItem.asrOptions || {};+ const language = asrOpts.language || uploadLanguage.value;+ if (language) {+ formData.append('language', language);+ }- // Add tags if selected- const tagsToUse = nextFileItem.tags || selectedTags.value || [];- tagsToUse.forEach((tag, index) => {- const tagId = tag.id || tag;- formData.append(`tag_ids[${index}]`, tagId);- });+ if (connectorSupportsDiarization.value) {+ const minSpeakers = asrOpts.min_speakers || uploadMinSpeakers.value;+ const maxSpeakers = asrOpts.max_speakers || uploadMaxSpeakers.value;- // Add folder if selected- const folderToUse = nextFileItem.folder_id || selectedFolderId.value;- if (folderToUse) {- formData.append('folder_id', folderToUse);+ if (minSpeakers && minSpeakers !== '') {+ formData.append('min_speakers', minSpeakers.toString());}-- // Add ASR options (language is always sent, speaker options only if supported)- const asrOpts = nextFileItem.asrOptions || {};- const language = asrOpts.language || uploadLanguage.value;- if (language) {- formData.append('language', language);+ if (maxSpeakers && maxSpeakers !== '') {+ formData.append('max_speakers', maxSpeakers.toString());}+ }- if (connectorSupportsDiarization.value) {- const minSpeakers = asrOpts.min_speakers || uploadMinSpeakers.value;- const maxSpeakers = asrOpts.max_speakers || uploadMaxSpeakers.value;+ // Use XMLHttpRequest for per-file upload progress+ const data = await new Promise((resolve, reject) => {+ const xhr = new XMLHttpRequest();- if (minSpeakers && minSpeakers !== '') {- formData.append('min_speakers', minSpeakers.toString());+ xhr.upload.onprogress = (e) => {+ if (e.lengthComputable) {+ // Map upload progress to 5-90% range+ fileItem.progress = Math.round(5 + (e.loaded / e.total) * 85);}- if (maxSpeakers && maxSpeakers !== '') {- formData.append('max_speakers', maxSpeakers.toString());+ };++ xhr.onload = () => {+ const contentType = xhr.getResponseHeader('content-type') || '';+ if (!contentType.includes('application/json')) {+ const titleMatch = xhr.responseText.match(/<title>([^<]+)<\/title>/i);+ const h1Match = xhr.responseText.match(/<h1>([^<]+)<\/h1>/i);+ reject(new Error(titleMatch?.[1] || h1Match?.[1] ||+ `Server error (${xhr.status}): Response was not JSON`));+ return;}- }- processingMessage.value = 'Uploading file...';- processingProgress.value = 10;+ let parsed;+ try {+ parsed = JSON.parse(xhr.responseText);+ } catch {+ reject(new Error(`Invalid JSON response (${xhr.status})`));+ return;+ }- const response = await fetch('/upload', { method: 'POST', body: formData });+ if (xhr.status === 202 && parsed.id) {+ resolve(parsed);+ } else if (!String(xhr.status).startsWith('2')) {+ let errorMsg = parsed.error || `Upload failed with status ${xhr.status}`;+ if (xhr.status === 413) errorMsg = parsed.error || `File too large. Max: ${parsed.max_size_mb?.toFixed(0) || maxFileSizeMB.value} MB.`;+ reject(new Error(errorMsg));+ } else {+ reject(new Error('Unexpected success response from server after upload.'));+ }+ };- // Safely parse JSON response, handling HTML error pages- let data;- const contentType = response.headers.get('content-type') || '';- if (!contentType.includes('application/json')) {- const text = await response.text();- const titleMatch = text.match(/<title>([^<]+)<\/title>/i);- const h1Match = text.match(/<h1>([^<]+)<\/h1>/i);- throw new Error(titleMatch?.[1] || h1Match?.[1] ||- `Server error (${response.status}): Response was not JSON`);- }- data = await response.json();+ xhr.onerror = () => reject(new Error('Network error during upload'));+ xhr.ontimeout = () => reject(new Error('Upload timed out'));++ // Store abort controller on item for cancellation+ fileItem._xhr = xhr;- if (!response.ok) {- let errorMsg = data.error || `Upload failed with status ${response.status}`;- if (response.status === 413) errorMsg = data.error || `File too large. Max: ${data.max_size_mb?.toFixed(0) || maxFileSizeMB.value} MB.`;- throw new Error(errorMsg);+ xhr.open('POST', '/upload');+ // Include CSRF token (required for POST requests)+ const csrfToken = document.querySelector('meta[name="csrf-token"]')?.getAttribute('content');+ if (csrfToken) {+ xhr.setRequestHeader('X-CSRFToken', csrfToken);}+ xhr.send(formData);+ });- if (response.status === 202 && data.id) {- console.log(`File ${nextFileItem.file.name} uploaded. Recording ID: ${data.id}. Starting status poll.`);- nextFileItem.status = 'pending';- nextFileItem.recordingId = data.id;- processingMessage.value = 'Upload complete. Waiting for processing...';- processingProgress.value = 30;+ // Upload succeeded - recording is now on the server+ console.log(`File ${fileItem.file.name} uploaded. Recording ID: ${data.id}. Server will process via job queue.`);+ fileItem.status = 'pending';+ fileItem.recordingId = data.id;+ fileItem.progress = 100;++ // Add to recordings list+ recordings.value.unshift(data);+ totalRecordings.value++;++ // Handle duplicate warning+ if (data.duplicate_warning) {+ const warning = data.duplicate_warning;+ const existingDate = warning.existing_created_at+ ? new Date(warning.existing_created_at).toLocaleDateString()+ : '';+ const existingName = warning.existing_title || 'Unknown';+ showToast(+ `⚠️ ${existingName} (${existingDate})`,+ 'fa-copy'+ );+ fileItem.duplicateWarning = warning;+ }- recordings.value.unshift(data);- totalRecordings.value++;- pollProcessingStatus(nextFileItem);+ } catch (error) {+ console.error(`Upload Error for ${fileItem.file.name} (Client ID: ${fileItem.clientId}):`, error);+ fileItem.status = 'failed';+ fileItem.error = error.message;+ fileItem.progress = 0;- } else {- throw new Error('Unexpected success response from server after upload.');- }+ // Show friendly error message+ const friendlyErr = getFriendlyError(error.message);+ setGlobalError(`${friendlyErr.title}: ${friendlyErr.guidance}`);- } catch (error) {- console.error(`Upload/Processing Error for ${nextFileItem.file.name} (Client ID: ${nextFileItem.clientId}):`, error);- nextFileItem.status = 'failed';- nextFileItem.error = error.message;- const failedRecordIndex = recordings.value.findIndex(r => r.id === nextFileItem.recordingId);- if (failedRecordIndex !== -1) {- recordings.value[failedRecordIndex].status = 'FAILED';- recordings.value[failedRecordIndex].transcription = `Upload/Processing failed: ${error.message}`;- } else {- // Show friendly error message in toast- const friendlyErr = getFriendlyError(error.message);- setGlobalError(`${friendlyErr.title}: ${friendlyErr.guidance}`);- }+ // Store failed upload in IndexedDB for background sync retry+ try {+ await FailedUploads.storeFailedUpload({+ file: fileItem.file,+ fileName: fileItem.file.name,+ fileSize: fileItem.file.size,+ clientId: fileItem.clientId,+ notes: fileItem.notes,+ tags: fileItem.tags,+ asrOptions: fileItem.asrOptions,+ error: error.message+ });- // Store failed upload in IndexedDB for background sync retry- try {- await FailedUploads.storeFailedUpload({- file: nextFileItem.file,- fileName: nextFileItem.file.name,- fileSize: nextFileItem.file.size,- clientId: nextFileItem.clientId,- notes: nextFileItem.notes,- tags: nextFileItem.tags,- asrOptions: nextFileItem.asrOptions,- error: error.message- });-- // Register for background sync- if ('serviceWorker' in navigator && 'sync' in ServiceWorkerRegistration.prototype) {- const registration = await navigator.serviceWorker.ready;- await registration.sync.register('sync-uploads');- console.log('[Upload] Registered background sync for failed upload');- showToast('Upload will retry automatically when connection is restored', 'info');- }- } catch (syncError) {- console.warn('[Upload] Failed to register background sync:', syncError);+ if ('serviceWorker' in navigator && 'sync' in ServiceWorkerRegistration.prototype) {+ const registration = await navigator.serviceWorker.ready;+ await registration.sync.register('sync-uploads');+ console.log('[Upload] Registered background sync for failed upload');}-- resetCurrentFileProcessingState();- isProcessingActive.value = false;- await Vue.nextTick();- startProcessingQueue();+ } catch (syncError) {+ console.warn('[Upload] Failed to register background sync:', syncError);}- } else {- console.log("Upload queue is empty or no files are queued.");- isProcessingActive.value = false;+ } finally {+ fileItem._xhr = null;+ releaseUploadSlot();}};- const startStatusPolling = (fileItem, recordingId) => {- fileItem.recordingId = recordingId;- pollProcessingStatus(fileItem);- };-- const pollProcessingStatus = (fileItem) => {- if (pollInterval.value) clearInterval(pollInterval.value);-- const recordingId = fileItem.recordingId;- if (!recordingId) {- console.error("Cannot poll status without recording ID for", fileItem.file.name);- fileItem.status = 'failed';- fileItem.error = 'Internal error: Missing recording ID for polling.';- resetCurrentFileProcessingState();- isProcessingActive.value = false;- Vue.nextTick(startProcessingQueue);+ /**+ * Start uploading all ready files in parallel (with concurrency limit).+ * Processing status is tracked via allJobs polling in app.modular.js.+ */+ const startProcessingQueue = async () => {+ const readyItems = uploadQueue.value.filter(item => item.status === 'ready');+ if (readyItems.length === 0) {+ console.log("No files ready to upload.");return;}- processingMessage.value = 'Waiting for transcription...';- processingProgress.value = 40;-- pollInterval.value = setInterval(async () => {- const shouldStopPolling = !currentlyProcessingFile.value ||- currentlyProcessingFile.value.clientId !== fileItem.clientId ||- fileItem.status === 'failed' ||- (fileItem.status === 'completed' && (!fileItem.willAutoSummarize || fileItem.summaryCompleted));-- if (shouldStopPolling) {- console.log(`Polling stopped for ${fileItem.clientId} as it's no longer active or finished.`);- clearInterval(pollInterval.value);- pollInterval.value = null;- if (currentlyProcessingFile.value && currentlyProcessingFile.value.clientId === fileItem.clientId) {- resetCurrentFileProcessingState();- isProcessingActive.value = false;- await Vue.nextTick();- startProcessingQueue();- }- return;- }-- try {- console.log(`Polling status for recording ID: ${recordingId} (${fileItem.file.name})`);- // Use lightweight status-only endpoint- const response = await fetch(`/recording/${recordingId}/status`);- if (!response.ok) throw new Error(`Status check failed with status ${response.status}`);-- // Safely parse JSON response, handling HTML error pages- const statusContentType = response.headers.get('content-type') || '';- if (!statusContentType.includes('application/json')) {- const text = await response.text();- const titleMatch = text.match(/<title>([^<]+)<\/title>/i);- const h1Match = text.match(/<h1>([^<]+)<\/h1>/i);- throw new Error(titleMatch?.[1] || h1Match?.[1] ||- `Server error (${response.status}): Status response was not JSON`);- }- const statusData = await response.json();- const galleryIndex = recordings.value.findIndex(r => r.id === recordingId);-- // Update status in recordings list- if (galleryIndex !== -1) {- // Create new object to ensure Vue reactivity- recordings.value[galleryIndex] = {- ...recordings.value[galleryIndex],- status: statusData.status- };-- // Update selectedRecording with new object reference for reactivity- if (selectedRecording.value?.id === recordingId) {- selectedRecording.value = {- ...selectedRecording.value,- status: statusData.status- };- }- }-- const previousStatus = fileItem.status;- fileItem.status = statusData.status;-- if (statusData.status === 'COMPLETED') {- // Fetch full recording data when complete- const fullResponse = await fetch(`/api/recordings/${recordingId}`);- if (fullResponse.ok) {- const data = await fullResponse.json();-- // Update recordings list first- if (galleryIndex !== -1) {- recordings.value[galleryIndex] = data;- }-- // Always update selectedRecording if it's the current recording,- // even if it's not in the current recordings list (e.g., filtered out)- if (selectedRecording.value?.id === recordingId) {- selectedRecording.value = data;- // Force Vue to detect the change- await nextTick();- }-- // Store display name separately since File.name is read-only- fileItem.displayName = data.title || data.original_filename || fileItem.file.name;- }-- console.log(`Processing COMPLETED for ${fileItem.displayName} (ID: ${recordingId})`);-- if (previousStatus === 'summarizing') {- console.log(`Auto-summary completed for ${fileItem.displayName}`);- processingMessage.value = 'Processing complete!';- processingProgress.value = 100;- fileItem.status = 'completed';- fileItem.summaryCompleted = true;-- clearInterval(pollInterval.value);- pollInterval.value = null;- resetCurrentFileProcessingState();- isProcessingActive.value = false;- // Refresh token budget after LLM operations- if (onChatComplete) onChatComplete();-- startProcessingQueue();- return;- } else if (fileItem.willAutoSummarize && !fileItem.hasCheckedForAutoSummary) {- processingMessage.value = 'Transcription complete!';- processingProgress.value = 85;- fileItem.status = 'awaiting_summary';- fileItem.hasCheckedForAutoSummary = true;- fileItem.autoSummaryStartTime = Date.now();- return;- } else if (fileItem.willAutoSummarize && fileItem.hasCheckedForAutoSummary) {- const waitTime = Date.now() - fileItem.autoSummaryStartTime;- const maxWaitTime = 5000; // 5 seconds - if no summarization starts, complete-- if (waitTime > maxWaitTime) {- console.log(`Auto-summary did not start within ${maxWaitTime}ms, completing`);- processingMessage.value = 'Processing complete!';- processingProgress.value = 100;- fileItem.status = 'completed';- fileItem.summaryCompleted = false; // Summary didn't happen- clearInterval(pollInterval.value);- pollInterval.value = null;- resetCurrentFileProcessingState();- isProcessingActive.value = false;- // Refresh token budget after LLM operations- if (onChatComplete) onChatComplete();- startProcessingQueue();- return;- }- return;- } else {- processingMessage.value = 'Processing complete!';- processingProgress.value = 100;- fileItem.status = 'completed';- fileItem.summaryCompleted = true;-- clearInterval(pollInterval.value);- pollInterval.value = null;- resetCurrentFileProcessingState();- isProcessingActive.value = false;- // Refresh token budget after LLM operations- if (onChatComplete) onChatComplete();- startProcessingQueue();- return;- }-- } else if (statusData.status === 'FAILED') {- console.log(`Processing FAILED for ${fileItem.displayName || fileItem.file.name} (ID: ${recordingId})`);- processingMessage.value = 'Processing failed.';- processingProgress.value = 100;- fileItem.status = 'failed';+ isProcessingActive.value = true;+ console.log(`Starting parallel upload of ${readyItems.length} file(s) (max ${MAX_CONCURRENT_UPLOADS} concurrent)...`);++ // Fire off all uploads concurrently (semaphore handles limiting)+ const uploadPromises = readyItems.map(item => uploadSingleFile(item));+ // Don't await - let them run in background. isProcessingActive is cleared by releaseUploadSlot.+ Promise.allSettled(uploadPromises).then(() => {+ console.log('All uploads settled.');+ });+ };- // Fetch full data to get error details and update UI- try {- const failedResponse = await fetch(`/api/recordings/${recordingId}`);- if (failedResponse.ok) {- const failedData = await failedResponse.json();- fileItem.error = failedData.error_message || 'Processing failed on server.';-- // Update recordings list so error shows in detail view- const galleryIndex = recordings.value.findIndex(r => r.id === recordingId);- if (galleryIndex !== -1) {- recordings.value[galleryIndex] = failedData;- }-- // Update selectedRecording to show error in transcription panel- if (selectedRecording.value?.id === recordingId) {- selectedRecording.value = failedData;- await nextTick();- }- } else {- fileItem.error = 'Processing failed on server.';- }- } catch (err) {- fileItem.error = 'Processing failed on server.';- }+ // Keep backward-compat aliases+ const startStatusPolling = (fileItem, recordingId) => {+ // No longer needed - allJobs polling handles status tracking+ fileItem.recordingId = recordingId;+ };- // Show friendly error message in toast- const friendlyErr = getFriendlyError(fileItem.error);- setGlobalError(`${friendlyErr.title}: ${friendlyErr.guidance}`);- clearInterval(pollInterval.value);- pollInterval.value = null;- resetCurrentFileProcessingState();- isProcessingActive.value = false;- await Vue.nextTick();- startProcessingQueue();-- } else if (statusData.status === 'PROCESSING') {- const couldUseChunking = chunkingEnabled.value && !useAsrEndpoint.value;-- if (couldUseChunking) {- if (chunkingMode.value === 'size') {- const chunkThresholdBytes = chunkingLimit.value * 1024 * 1024;- const willUseChunking = fileItem.file.size > chunkThresholdBytes;-- if (willUseChunking) {- processingMessage.value = 'Processing large file (chunking in progress)...';- const maxProgress = fileItem.willAutoSummarize ? 70 : 80;- processingProgress.value = Math.round(Math.min(maxProgress, processingProgress.value + Math.random() * 3));- } else {- processingMessage.value = 'Transcription in progress...';- const maxProgress = fileItem.willAutoSummarize ? 65 : 75;- processingProgress.value = Math.round(Math.min(maxProgress, processingProgress.value + Math.random() * 5));- }- } else {- processingMessage.value = 'Processing file (chunking determined server-side)...';- const maxProgress = fileItem.willAutoSummarize ? 70 : 80;- processingProgress.value = Math.round(Math.min(maxProgress, processingProgress.value + Math.random() * 3));- }- } else {- processingMessage.value = 'Transcription in progress...';- const maxProgress = fileItem.willAutoSummarize ? 65 : 75;- processingProgress.value = Math.round(Math.min(maxProgress, processingProgress.value + Math.random() * 5));- }- } else if (statusData.status === 'SUMMARIZING') {- console.log(`Auto-summary started for ${fileItem.displayName || fileItem.file.name}`);- processingMessage.value = 'Generating summary...';- processingProgress.value = 90;- fileItem.status = 'summarizing';- } else {- processingMessage.value = 'Waiting in queue...';- processingProgress.value = 45;- }- } catch (error) {- console.error(`Polling Error for ${fileItem.displayName || fileItem.file.name} (ID: ${recordingId}):`, error);- fileItem.status = 'failed';- fileItem.error = `Error checking status: ${error.message}`;- setGlobalError(`Error checking status for "${fileItem.displayName || fileItem.file.name}": ${error.message}.`);- const galleryIndex = recordings.value.findIndex(r => r.id === recordingId);- if (galleryIndex !== -1) recordings.value[galleryIndex].status = 'FAILED';-- clearInterval(pollInterval.value);- pollInterval.value = null;- resetCurrentFileProcessingState();- isProcessingActive.value = false;- await Vue.nextTick();- startProcessingQueue();- }- }, 5000);+ const pollProcessingStatus = () => {+ // No-op: status tracking is now handled by allJobs polling in app.modular.js};// Tag selection helpersdiff --git a/static/locales/de.json b/static/locales/de.jsonindex d5a5ec7d..d3d1bd06 100644--- a/static/locales/de.json+++ b/static/locales/de.json@@ -1144,7 +1144,8 @@"transcribing": "Transkribiere...","untitled": "Unbenannte Aufnahme","uploadProgress": "Upload-Fortschritt",- "willAutoSummarize": "Wird nach der Transkription automatisch zusammengefasst"+ "willAutoSummarize": "Wird nach der Transkription automatisch zusammengefasst",+ "duplicateDetected": "Diese Datei scheint ein Duplikat von \"{{existingName}}\" zu sein (hochgeladen am {{existingDate}})"},"uploadProgress": {"title": "Upload-Fortschritt"diff --git a/static/locales/en.json b/static/locales/en.jsonindex dda9c25e..8ea7c22d 100644--- a/static/locales/en.json+++ b/static/locales/en.json@@ -296,7 +296,6 @@"currentUsageMinutes": "Current Usage","userTranscriptionUsage": "User Transcription Usage (This Month)","noTranscriptionData": "No transcription usage data available",- "minutes": "minutes","percentUsed": "used","topUsers": "Top Users","topUsersByStorage": "Top Users by Storage",@@ -403,20 +402,56 @@"title": "Choose Color Scheme","themes": {"light": {- "blue": { "name": "Ocean Blue", "description": "Classic blue theme with professional appeal" },- "emerald": { "name": "Forest Emerald", "description": "Fresh green theme for a natural feel" },- "purple": { "name": "Royal Purple", "description": "Elegant purple theme with sophistication" },- "rose": { "name": "Sunset Rose", "description": "Warm pink theme with gentle energy" },- "amber": { "name": "Golden Amber", "description": "Warm yellow theme for brightness" },- "teal": { "name": "Ocean Teal", "description": "Cool teal theme for tranquility" }+ "blue": {+ "name": "Ocean Blue",+ "description": "Classic blue theme with professional appeal"+ },+ "emerald": {+ "name": "Forest Emerald",+ "description": "Fresh green theme for a natural feel"+ },+ "purple": {+ "name": "Royal Purple",+ "description": "Elegant purple theme with sophistication"+ },+ "rose": {+ "name": "Sunset Rose",+ "description": "Warm pink theme with gentle energy"+ },+ "amber": {+ "name": "Golden Amber",+ "description": "Warm yellow theme for brightness"+ },+ "teal": {+ "name": "Ocean Teal",+ "description": "Cool teal theme for tranquility"+ }},"dark": {- "blue": { "name": "Midnight Blue", "description": "Deep blue theme for focused work" },- "emerald": { "name": "Dark Forest", "description": "Rich green theme for comfortable viewing" },- "purple": { "name": "Deep Purple", "description": "Mysterious purple theme for creativity" },- "rose": { "name": "Dark Rose", "description": "Muted pink theme with subtle warmth" },- "amber": { "name": "Dark Amber", "description": "Warm brown theme for cozy sessions" },- "teal": { "name": "Deep Teal", "description": "Dark teal theme for calm focus" }+ "blue": {+ "name": "Midnight Blue",+ "description": "Deep blue theme for focused work"+ },+ "emerald": {+ "name": "Dark Forest",+ "description": "Rich green theme for comfortable viewing"+ },+ "purple": {+ "name": "Deep Purple",+ "description": "Mysterious purple theme for creativity"+ },+ "rose": {+ "name": "Dark Rose",+ "description": "Muted pink theme with subtle warmth"+ },+ "amber": {+ "name": "Dark Amber",+ "description": "Warm brown theme for cozy sessions"+ },+ "teal": {+ "name": "Deep Teal",+ "description": "Dark teal theme for calm focus"+ }}}},@@ -703,7 +738,7 @@"speakerRequirement": "Speaker identification requires recordings with multiple speakers","activeFilters": "Active Filters:","tagsCount": "tags",- "speakersCount": "speakers",+ "speakersCount": "speakers","dateRangeActive": "Date range active","filtersActive": "Filters active","askQuestions": "Ask Questions About Your Transcriptions",@@ -964,7 +999,6 @@"searchRecordings": "Search recordings...","sharedWithMe": "Shared with Me","starred": "Starred",- "inbox": "Inbox","sortBy": "Sort By","sortByDate": "Created Date","sortByMeetingDate": "Meeting Date",@@ -1161,7 +1195,8 @@"transcribing": "Transcribing...","untitled": "Untitled Recording","uploadProgress": "Upload Progress",- "willAutoSummarize": "Will auto-summarize after transcription"+ "willAutoSummarize": "Will auto-summarize after transcription",+ "duplicateDetected": "This file appears to be a duplicate of \"{{existingName}}\" (uploaded {{existingDate}})"},"uploadProgress": {"title": "Upload Progress"@@ -1270,4 +1305,4 @@"recordingData": "Recording Data","contentSections": "Content Sections"}-}\ No newline at end of file+}diff --git a/static/locales/es.json b/static/locales/es.jsonindex 5e20c9c0..2977513c 100644--- a/static/locales/es.json+++ b/static/locales/es.json@@ -1144,7 +1144,8 @@"transcribing": "Transcribiendo...","untitled": "Grabación Sin Título","uploadProgress": "Progreso de Carga",- "willAutoSummarize": "Se resumirá automáticamente después de la transcripción"+ "willAutoSummarize": "Se resumirá automáticamente después de la transcripción",+ "duplicateDetected": "Este archivo parece ser un duplicado de \"{{existingName}}\" (subido el {{existingDate}})"},"uploadProgress": {"title": "Progreso de Carga"diff --git a/static/locales/fr.json b/static/locales/fr.jsonindex f137351e..fc885340 100644--- a/static/locales/fr.json+++ b/static/locales/fr.json@@ -1138,7 +1138,8 @@"transcribing": "Transcription...","untitled": "Enregistrement Sans Titre","uploadProgress": "Progression du Téléchargement",- "willAutoSummarize": "Résumé automatique après la transcription"+ "willAutoSummarize": "Résumé automatique après la transcription",+ "duplicateDetected": "Ce fichier semble être un doublon de \"{{existingName}}\" (importé le {{existingDate}})"},"uploadProgress": {"title": "Progression du Téléchargement"diff --git a/static/locales/ru.json b/static/locales/ru.jsonindex ae717ea9..57e1996c 100644--- a/static/locales/ru.json+++ b/static/locales/ru.json@@ -1160,7 +1160,8 @@"transcribing": "Транскрибирование...","untitled": "Запись без названия","uploadProgress": "Прогресс загрузки",- "willAutoSummarize": "Автоматически создаст резюме после транскрибирования"+ "willAutoSummarize": "Автоматически создаст резюме после транскрибирования",+ "duplicateDetected": "Этот файл является дубликатом \"{{existingName}}\" (загружен {{existingDate}})"},"uploadProgress": {"title": "Прогресс загрузки"diff --git a/static/locales/zh.json b/static/locales/zh.jsonindex c32ca8a8..b2b13e20 100644--- a/static/locales/zh.json+++ b/static/locales/zh.json@@ -1273,7 +1273,8 @@"transcribing": "转录中...","untitled": "无标题录音","uploadProgress": "上传进度",- "willAutoSummarize": "转录后将自动生成摘要"+ "willAutoSummarize": "转录后将自动生成摘要",+ "duplicateDetected": "此文件似乎与 \"{{existingName}}\" 重复(上传于 {{existingDate}})"},"uploadProgress": {"title": "上传进度"diff --git a/templates/components/progress-popup.html b/templates/components/progress-popup.htmlindex e38c9c9e..f8a68795 100644--- a/templates/components/progress-popup.html+++ b/templates/components/progress-popup.html@@ -173,6 +173,7 @@class="px-2 py-1 bg-[var(--bg-tertiary)]/30 rounded-md flex items-center gap-2 hover:bg-[var(--bg-tertiary)]/50 cursor-pointer mb-1"><i class="fas fa-check-circle text-xs text-green-500"></i><span class="text-xs truncate flex-1 text-[var(--text-primary)]">${item.title}</span>+ <i v-if="item.duplicateWarning" class="fas fa-copy text-xs text-amber-500" title="Duplicate file detected"></i><span class="text-[10px] px-1.5 py-0.5 rounded bg-green-500/20 text-green-500 font-medium">Done</span></div></div>