Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python

Add parallel uploads with concurrency limit and SHA-256 duplicate detection

Murtaza Nasir9134a9e1e7f9
diff --git a/src/api/recordings.py b/src/api/recordings.py
index c3c301e1..574ce457 100644
--- a/src/api/recordings.py
+++ b/src/api/recordings.py
@@ -37,6 +37,7 @@ from src.services.embeddings import process_recording_chunks
from src.file_exporter import export_recording, mark_export_as_deleted
from src.utils.ffprobe import get_codec_info, get_creation_date, FFProbeError
from src.utils.audio_conversion import convert_if_needed
+from src.utils.file_hash import compute_file_sha256
# Create blueprint
recordings_bp = Blueprint('recordings', __name__)
@@ -1978,6 +1979,27 @@ def upload_file():
# Get final file size (of original or converted file)
final_file_size = os.path.getsize(filepath)
+ # Compute file hash for duplicate detection (hash the final/converted file)
+ file_hash = None
+ duplicate_warning = None
+ try:
+ file_hash = compute_file_sha256(filepath)
+ existing = Recording.query.filter_by(
+ user_id=current_user.id, file_hash=file_hash
+ ).first()
+ if existing:
+ duplicate_warning = {
+ 'existing_recording_id': existing.id,
+ 'existing_title': existing.title,
+ 'existing_created_at': existing.created_at.isoformat() if existing.created_at else None
+ }
+ current_app.logger.info(
+ f"Duplicate file detected for user {current_user.id}: "
+ f"hash={file_hash[:12]}... matches recording {existing.id}"
+ )
+ except Exception as e:
+ current_app.logger.warning(f"Could not compute file hash: {e}")
+
# Determine MIME type of the final file
mime_type, _ = mimetypes.guess_type(filepath)
current_app.logger.info(f"Final MIME type: {mime_type} for file {filepath}")
@@ -2119,7 +2141,8 @@ def upload_file():
mime_type=mime_type,
notes=notes,
folder_id=selected_folder.id if selected_folder else None,
- processing_source='upload' # Track that this was manually uploaded
+ processing_source='upload', # Track that this was manually uploaded
+ file_hash=file_hash
)
db.session.add(recording)
db.session.commit()
@@ -2160,7 +2183,10 @@ def upload_file():
)
current_app.logger.info(f"Transcription job queued for recording ID: {recording.id}")
- return jsonify(recording.to_dict(viewer_user=current_user)), 202
+ response_data = recording.to_dict(viewer_user=current_user)
+ if duplicate_warning:
+ response_data['duplicate_warning'] = duplicate_warning
+ return jsonify(response_data), 202
except RequestEntityTooLarge:
max_size_mb = current_app.config['MAX_CONTENT_LENGTH'] / (1024 * 1024)
diff --git a/src/file_monitor.py b/src/file_monitor.py
index 10f7b6ba..f5d5f86c 100644
--- a/src/file_monitor.py
+++ b/src/file_monitor.py
@@ -361,6 +361,14 @@ class FileMonitor:
self.logger.error(f"FFmpeg conversion failed: {e}")
raise
+ # Compute file hash for duplicate detection
+ file_hash = None
+ try:
+ from src.utils.file_hash import compute_file_sha256
+ file_hash = compute_file_sha256(str(final_path))
+ except Exception as e:
+ self.logger.warning(f"Could not compute file hash: {e}")
+
# Get file size and MIME type
file_size = final_path.stat().st_size
mime_type, _ = mimetypes.guess_type(str(final_path))
@@ -376,6 +384,16 @@ class FileMonitor:
meeting_date = now
self.logger.debug("No metadata creation date found, using current time")
+ # Check for duplicate
+ if file_hash:
+ existing = Recording.query.filter_by(user_id=user_id, file_hash=file_hash).first()
+ if existing:
+ self.logger.warning(
+ f"Duplicate file detected for user {user_id}: "
+ f"hash={file_hash[:12]}... matches recording {existing.id} "
+ f"({existing.title}). Processing anyway."
+ )
+
recording = Recording(
audio_path=str(final_path),
original_filename=original_filename,
@@ -386,7 +404,8 @@ class FileMonitor:
user_id=user_id,
mime_type=mime_type,
is_inbox=True, # Auto-processed files go to inbox
- processing_source='auto_process' # Track that this was auto-processed
+ processing_source='auto_process', # Track that this was auto-processed
+ file_hash=file_hash
)
db.session.add(recording)
diff --git a/src/init_db.py b/src/init_db.py
index 59f448cf..01a6867a 100644
--- a/src/init_db.py
+++ b/src/init_db.py
@@ -517,6 +517,15 @@ def initialize_database(app):
except Exception as e:
app.logger.warning(f"Could not create unique index on user.sso_subject: {e}")
+ # Add file_hash column for duplicate detection
+ if add_column_if_not_exists(engine, 'recording', 'file_hash', 'VARCHAR(64)'):
+ app.logger.info("Added file_hash column to recording table")
+ try:
+ if create_index_if_not_exists(engine, 'ix_recording_user_file_hash', 'recording', 'user_id, file_hash'):
+ app.logger.info("Created index ix_recording_user_file_hash on recording (user_id, file_hash)")
+ except Exception as e:
+ app.logger.warning(f"Could not create index on recording (user_id, file_hash): {e}")
+
# Add folder_id column to recording table for folders feature
if add_column_if_not_exists(engine, 'recording', 'folder_id', 'INTEGER'):
app.logger.info("Added folder_id column to recording table")
diff --git a/src/models/recording.py b/src/models/recording.py
index 225b99e5..8ddb5a2f 100644
--- a/src/models/recording.py
+++ b/src/models/recording.py
@@ -41,6 +41,7 @@ class Recording(db.Model):
summarization_duration_seconds = db.Column(db.Integer, nullable=True) # Time taken for summarization
processing_source = db.Column(db.String(50), default='upload') # upload, auto_process, recording
error_message = db.Column(db.Text, nullable=True) # Store detailed error messages
+ file_hash = db.Column(db.String(64), nullable=True) # SHA-256 hash for duplicate detection
# Auto-deletion and archival fields
audio_deleted_at = db.Column(db.DateTime, nullable=True) # When audio file was deleted (null = not deleted)
diff --git a/src/utils/file_hash.py b/src/utils/file_hash.py
new file mode 100644
index 00000000..7dab3860
--- /dev/null
+++ b/src/utils/file_hash.py
@@ -0,0 +1,24 @@
+"""File hashing utility for duplicate detection."""
+
+import hashlib
+
+
+def compute_file_sha256(filepath, chunk_size=8192):
+ """
+ Compute SHA-256 hash of a file, reading in chunks to handle large files.
+
+ Args:
+ filepath: Path to the file to hash
+ chunk_size: Size of chunks to read at a time (default 8KB)
+
+ Returns:
+ 64-character hex digest string
+ """
+ sha256 = hashlib.sha256()
+ with open(filepath, 'rb') as f:
+ while True:
+ chunk = f.read(chunk_size)
+ if not chunk:
+ break
+ sha256.update(chunk)
+ return sha256.hexdigest()
diff --git a/static/js/app.modular.js b/static/js/app.modular.js
index 76f22be6..60aa63e6 100644
--- a/static/js/app.modular.js
+++ b/static/js/app.modular.js
@@ -1634,6 +1634,63 @@ document.addEventListener('DOMContentLoaded', async () => {
}
});
+ // Track completed recording IDs to detect new completions
+ const completedRecordingIds = new Set();
+
+ // Watch allJobs for completed/failed transitions - update local recordings state
+ watch(allJobs, async (jobs) => {
+ for (const job of jobs) {
+ if (job.job_status === 'completed' && !completedRecordingIds.has(job.recording_id)) {
+ completedRecordingIds.add(job.recording_id);
+ try {
+ const fullResponse = await fetch(`/api/recordings/${job.recording_id}`);
+ if (fullResponse.ok) {
+ const data = await fullResponse.json();
+ const idx = recordings.value.findIndex(r => r.id === job.recording_id);
+ if (idx !== -1) {
+ recordings.value[idx] = data;
+ }
+ if (selectedRecording.value?.id === job.recording_id) {
+ selectedRecording.value = data;
+ }
+ // Update display name on upload queue item
+ const queueItem = uploadQueue.value.find(u => u.recordingId === job.recording_id);
+ if (queueItem) {
+ queueItem.displayName = data.title || data.original_filename || queueItem.file?.name;
+ queueItem.status = 'completed';
+ }
+ // Refresh token budget
+ if (typeof loadTokenBudget === 'function') loadTokenBudget();
+ }
+ } catch (err) {
+ console.error(`Error fetching completed recording ${job.recording_id}:`, err);
+ }
+ } else if (job.job_status === 'failed' && !completedRecordingIds.has(`fail_${job.recording_id}`)) {
+ completedRecordingIds.add(`fail_${job.recording_id}`);
+ try {
+ const failedResponse = await fetch(`/api/recordings/${job.recording_id}`);
+ if (failedResponse.ok) {
+ const failedData = await failedResponse.json();
+ const idx = recordings.value.findIndex(r => r.id === job.recording_id);
+ if (idx !== -1) {
+ recordings.value[idx] = failedData;
+ }
+ if (selectedRecording.value?.id === job.recording_id) {
+ selectedRecording.value = failedData;
+ }
+ const queueItem = uploadQueue.value.find(u => u.recordingId === job.recording_id);
+ if (queueItem) {
+ queueItem.status = 'failed';
+ queueItem.error = failedData.error_message || 'Processing failed on server.';
+ }
+ }
+ } catch (err) {
+ console.error(`Error fetching failed recording ${job.recording_id}:`, err);
+ }
+ }
+ }
+ }, { deep: true });
+
// Get job details for a recording
const getJobDetails = (recordingId) => {
return jobQueueDetails.value[recordingId] || null;
@@ -1709,9 +1766,6 @@ document.addEventListener('DOMContentLoaded', async () => {
const unifiedProgressItems = computed(() => {
const items = new Map(); // Key by recordingId or clientId
- // Get the currently uploading file's clientId for special handling
- const currentUploadClientId = currentlyProcessingFile.value?.clientId;
-
// 1. First, add all backend jobs (these have the most accurate status)
for (const job of allJobs.value) {
const key = `rec_${job.recording_id}`;
@@ -1752,80 +1806,50 @@ document.addEventListener('DOMContentLoaded', async () => {
// 2. Add upload queue items (client-side tracking)
for (const upload of uploadQueue.value) {
- // Check if this is the currently uploading file
- const isCurrentUpload = upload.clientId === currentUploadClientId;
-
- // If we have a recordingId and it's already tracked from jobs, check if we should merge
+ // If we have a recordingId and it's already tracked from jobs, merge upload info
if (upload.recordingId) {
const key = `rec_${upload.recordingId}`;
const existing = items.get(key);
if (existing) {
- // Merge upload info into existing item
existing.clientId = upload.clientId;
existing.file = upload.file;
- // If this is the current upload, use the global progress refs
- if (isCurrentUpload && ['uploading', 'pending', 'processing', 'summarizing'].includes(upload.status)) {
- // Map upload status to unified status
- if (upload.status === 'uploading') {
- existing.status = 'uploading';
- } else if (upload.status === 'summarizing') {
- existing.status = 'summarizing';
- } else if (upload.status === 'processing' || upload.status === 'pending') {
- existing.status = 'transcribing';
- }
- existing.progress = processingProgress.value;
- existing.progressMessage = processingMessage.value || 'Processing...';
+ existing.duplicateWarning = upload.duplicateWarning || null;
+ // If still uploading, override job status with upload status
+ if (upload.status === 'uploading') {
+ existing.status = 'uploading';
+ existing.progress = upload.progress || 0;
+ existing.progressMessage = 'Uploading...';
existing.title = upload.displayName || upload.file?.name || existing.title;
}
continue;
}
}
- // Determine unified status from upload status
+ // Determine unified status from upload status (per-item progress)
let unifiedStatus = 'ready';
- let progressVal = 0;
+ let progressVal = upload.progress || 0;
let progressMsg = 'Waiting to upload...';
- // If this is the currently processing file, use global progress refs
- if (isCurrentUpload) {
- progressVal = processingProgress.value;
- progressMsg = processingMessage.value || 'Processing...';
-
- if (upload.status === 'uploading') {
- unifiedStatus = 'uploading';
- } else if (upload.status === 'pending' || upload.status === 'processing' || upload.status === 'PROCESSING') {
- unifiedStatus = 'transcribing';
- } else if (upload.status === 'summarizing' || upload.status === 'SUMMARIZING') {
- unifiedStatus = 'summarizing';
- } else if (upload.status === 'completed' || upload.status === 'COMPLETED') {
- unifiedStatus = 'completed';
- progressMsg = 'Done';
- } else if (upload.status === 'failed' || upload.status === 'FAILED') {
- unifiedStatus = 'failed';
- progressMsg = upload.error || 'Processing failed';
- } else if (upload.status === 'ready') {
- unifiedStatus = 'ready';
- progressMsg = 'Waiting to upload...';
- }
- } else {
- // Not the current upload - use item's own status
- if (upload.status === 'uploading') {
- unifiedStatus = 'uploading';
- progressMsg = 'Uploading...';
- } else if (upload.status === 'completed' || upload.status === 'COMPLETED') {
- unifiedStatus = 'completed';
- progressMsg = 'Done';
- } else if (upload.status === 'failed' || upload.status === 'FAILED') {
- unifiedStatus = 'upload_failed';
- progressMsg = upload.error || 'Upload failed';
- } else if (upload.status === 'ready') {
- unifiedStatus = 'ready';
- progressMsg = 'Waiting to upload...';
- } else if (upload.status === 'queued') {
- unifiedStatus = 'ready';
- progressMsg = 'Waiting to upload...';
- }
+ if (upload.status === 'uploading') {
+ unifiedStatus = 'uploading';
+ progressMsg = 'Uploading...';
+ } else if (upload.status === 'pending') {
+ unifiedStatus = 'queued';
+ progressVal = 100;
+ progressMsg = 'Uploaded, waiting for processing...';
+ } else if (upload.status === 'completed' || upload.status === 'COMPLETED') {
+ unifiedStatus = 'completed';
+ progressMsg = 'Done';
+ } else if (upload.status === 'failed' || upload.status === 'FAILED') {
+ unifiedStatus = 'upload_failed';
+ progressMsg = upload.error || 'Upload failed';
+ } else if (upload.status === 'ready') {
+ unifiedStatus = 'ready';
+ progressMsg = 'Waiting to upload...';
+ } else if (upload.status === 'queued') {
+ unifiedStatus = 'ready';
+ progressMsg = 'Waiting to upload...';
}
const key = upload.recordingId ? `rec_${upload.recordingId}` : `client_${upload.clientId}`;
@@ -1846,6 +1870,7 @@ document.addEventListener('DOMContentLoaded', async () => {
progressMessage: progressMsg,
queuePosition: null,
errorMessage: upload.status === 'failed' ? upload.error : null,
+ duplicateWarning: upload.duplicateWarning || null,
file: upload.file,
source: 'upload'
});
diff --git a/static/js/modules/composables/upload.js b/static/js/modules/composables/upload.js
index 7b033c0c..39f50f36 100644
--- a/static/js/modules/composables/upload.js
+++ b/static/js/modules/composables/upload.js
@@ -268,6 +268,39 @@ export function useUpload(state, utils) {
}
};
+ // --- Parallel Upload System ---
+ // Concurrency limiter: max 3 simultaneous uploads
+ const MAX_CONCURRENT_UPLOADS = 3;
+ let activeUploadCount = 0;
+ const pendingUploadQueue = []; // Functions waiting for a slot
+
+ const acquireUploadSlot = () => {
+ return new Promise(resolve => {
+ if (activeUploadCount < MAX_CONCURRENT_UPLOADS) {
+ activeUploadCount++;
+ resolve();
+ } else {
+ pendingUploadQueue.push(resolve);
+ }
+ });
+ };
+
+ const releaseUploadSlot = () => {
+ activeUploadCount--;
+ if (pendingUploadQueue.length > 0) {
+ activeUploadCount++;
+ const next = pendingUploadQueue.shift();
+ next();
+ }
+ // When all uploads are done, clear processing active flag
+ const stillUploading = uploadQueue.value.some(item =>
+ ['uploading', 'ready'].includes(item.status)
+ );
+ if (!stillUploading) {
+ isProcessingActive.value = false;
+ }
+ };
+
const resetCurrentFileProcessingState = () => {
if (pollInterval.value) clearInterval(pollInterval.value);
pollInterval.value = null;
@@ -276,424 +309,209 @@ export function useUpload(state, utils) {
processingMessage.value = '';
};
- const startProcessingQueue = async () => {
- console.log("Attempting to start processing queue...");
- if (isProcessingActive.value) {
- console.log("Queue processor already active.");
- return;
- }
+ /**
+ * Upload a single file to the server.
+ * Acquires a concurrency slot, uploads, then releases.
+ * Status updates are per-item (no global processingProgress).
+ */
+ const uploadSingleFile = async (fileItem) => {
+ await acquireUploadSlot();
- isProcessingActive.value = true;
- resetCurrentFileProcessingState();
+ fileItem.status = 'uploading';
+ fileItem.progress = 5;
- const nextFileItem = uploadQueue.value.find(item => item.status === 'ready' || item.status === 'queued');
+ try {
+ const formData = new FormData();
+ formData.append('file', fileItem.file);
- if (nextFileItem) {
- console.log(`Processing next file: ${nextFileItem.file.name} (Client ID: ${nextFileItem.clientId})`);
- currentlyProcessingFile.value = nextFileItem;
+ // Send file's lastModified timestamp for meeting_date
+ if (fileItem.file.lastModified) {
+ const lastModified = fileItem.file.lastModified;
+ formData.append('file_last_modified', lastModified.toString());
+ }
- // Check if this is a "reload" item (existing recording being tracked)
- if (nextFileItem.clientId.startsWith('reload-')) {
- console.log(`Skipping upload for existing recording: ${nextFileItem.recordingId}`);
- nextFileItem.status = 'processing';
- startStatusPolling(nextFileItem, nextFileItem.recordingId);
- return;
+ if (fileItem.notes) {
+ formData.append('notes', fileItem.notes);
}
- nextFileItem.status = 'uploading';
- processingMessage.value = 'Preparing upload...';
- processingProgress.value = 5;
+ // Add tags if selected
+ const tagsToUse = fileItem.tags || selectedTags.value || [];
+ tagsToUse.forEach((tag, index) => {
+ const tagId = tag.id || tag;
+ formData.append(`tag_ids[${index}]`, tagId);
+ });
- try {
- const formData = new FormData();
- formData.append('file', nextFileItem.file);
-
- // Send file's lastModified timestamp for meeting_date
- if (nextFileItem.file.lastModified) {
- const lastModified = nextFileItem.file.lastModified;
- formData.append('file_last_modified', lastModified.toString());
- console.log(`[Upload] File lastModified: ${lastModified} (${new Date(lastModified).toISOString()})`);
- }
+ // Add folder if selected
+ const folderToUse = fileItem.folder_id || selectedFolderId.value;
+ if (folderToUse) {
+ formData.append('folder_id', folderToUse);
+ }
- if (nextFileItem.notes) {
- formData.append('notes', nextFileItem.notes);
- }
+ // Add ASR options
+ const asrOpts = fileItem.asrOptions || {};
+ const language = asrOpts.language || uploadLanguage.value;
+ if (language) {
+ formData.append('language', language);
+ }
- // Add tags if selected
- const tagsToUse = nextFileItem.tags || selectedTags.value || [];
- tagsToUse.forEach((tag, index) => {
- const tagId = tag.id || tag;
- formData.append(`tag_ids[${index}]`, tagId);
- });
+ if (connectorSupportsDiarization.value) {
+ const minSpeakers = asrOpts.min_speakers || uploadMinSpeakers.value;
+ const maxSpeakers = asrOpts.max_speakers || uploadMaxSpeakers.value;
- // Add folder if selected
- const folderToUse = nextFileItem.folder_id || selectedFolderId.value;
- if (folderToUse) {
- formData.append('folder_id', folderToUse);
+ if (minSpeakers && minSpeakers !== '') {
+ formData.append('min_speakers', minSpeakers.toString());
}
-
- // Add ASR options (language is always sent, speaker options only if supported)
- const asrOpts = nextFileItem.asrOptions || {};
- const language = asrOpts.language || uploadLanguage.value;
- if (language) {
- formData.append('language', language);
+ if (maxSpeakers && maxSpeakers !== '') {
+ formData.append('max_speakers', maxSpeakers.toString());
}
+ }
- if (connectorSupportsDiarization.value) {
- const minSpeakers = asrOpts.min_speakers || uploadMinSpeakers.value;
- const maxSpeakers = asrOpts.max_speakers || uploadMaxSpeakers.value;
+ // Use XMLHttpRequest for per-file upload progress
+ const data = await new Promise((resolve, reject) => {
+ const xhr = new XMLHttpRequest();
- if (minSpeakers && minSpeakers !== '') {
- formData.append('min_speakers', minSpeakers.toString());
+ xhr.upload.onprogress = (e) => {
+ if (e.lengthComputable) {
+ // Map upload progress to 5-90% range
+ fileItem.progress = Math.round(5 + (e.loaded / e.total) * 85);
}
- if (maxSpeakers && maxSpeakers !== '') {
- formData.append('max_speakers', maxSpeakers.toString());
+ };
+
+ xhr.onload = () => {
+ const contentType = xhr.getResponseHeader('content-type') || '';
+ if (!contentType.includes('application/json')) {
+ const titleMatch = xhr.responseText.match(/<title>([^<]+)<\/title>/i);
+ const h1Match = xhr.responseText.match(/<h1>([^<]+)<\/h1>/i);
+ reject(new Error(titleMatch?.[1] || h1Match?.[1] ||
+ `Server error (${xhr.status}): Response was not JSON`));
+ return;
}
- }
- processingMessage.value = 'Uploading file...';
- processingProgress.value = 10;
+ let parsed;
+ try {
+ parsed = JSON.parse(xhr.responseText);
+ } catch {
+ reject(new Error(`Invalid JSON response (${xhr.status})`));
+ return;
+ }
- const response = await fetch('/upload', { method: 'POST', body: formData });
+ if (xhr.status === 202 && parsed.id) {
+ resolve(parsed);
+ } else if (!String(xhr.status).startsWith('2')) {
+ let errorMsg = parsed.error || `Upload failed with status ${xhr.status}`;
+ if (xhr.status === 413) errorMsg = parsed.error || `File too large. Max: ${parsed.max_size_mb?.toFixed(0) || maxFileSizeMB.value} MB.`;
+ reject(new Error(errorMsg));
+ } else {
+ reject(new Error('Unexpected success response from server after upload.'));
+ }
+ };
- // Safely parse JSON response, handling HTML error pages
- let data;
- const contentType = response.headers.get('content-type') || '';
- if (!contentType.includes('application/json')) {
- const text = await response.text();
- const titleMatch = text.match(/<title>([^<]+)<\/title>/i);
- const h1Match = text.match(/<h1>([^<]+)<\/h1>/i);
- throw new Error(titleMatch?.[1] || h1Match?.[1] ||
- `Server error (${response.status}): Response was not JSON`);
- }
- data = await response.json();
+ xhr.onerror = () => reject(new Error('Network error during upload'));
+ xhr.ontimeout = () => reject(new Error('Upload timed out'));
+
+ // Store abort controller on item for cancellation
+ fileItem._xhr = xhr;
- if (!response.ok) {
- let errorMsg = data.error || `Upload failed with status ${response.status}`;
- if (response.status === 413) errorMsg = data.error || `File too large. Max: ${data.max_size_mb?.toFixed(0) || maxFileSizeMB.value} MB.`;
- throw new Error(errorMsg);
+ xhr.open('POST', '/upload');
+ // Include CSRF token (required for POST requests)
+ const csrfToken = document.querySelector('meta[name="csrf-token"]')?.getAttribute('content');
+ if (csrfToken) {
+ xhr.setRequestHeader('X-CSRFToken', csrfToken);
}
+ xhr.send(formData);
+ });
- if (response.status === 202 && data.id) {
- console.log(`File ${nextFileItem.file.name} uploaded. Recording ID: ${data.id}. Starting status poll.`);
- nextFileItem.status = 'pending';
- nextFileItem.recordingId = data.id;
- processingMessage.value = 'Upload complete. Waiting for processing...';
- processingProgress.value = 30;
+ // Upload succeeded - recording is now on the server
+ console.log(`File ${fileItem.file.name} uploaded. Recording ID: ${data.id}. Server will process via job queue.`);
+ fileItem.status = 'pending';
+ fileItem.recordingId = data.id;
+ fileItem.progress = 100;
+
+ // Add to recordings list
+ recordings.value.unshift(data);
+ totalRecordings.value++;
+
+ // Handle duplicate warning
+ if (data.duplicate_warning) {
+ const warning = data.duplicate_warning;
+ const existingDate = warning.existing_created_at
+ ? new Date(warning.existing_created_at).toLocaleDateString()
+ : '';
+ const existingName = warning.existing_title || 'Unknown';
+ showToast(
+ `⚠️ ${existingName} (${existingDate})`,
+ 'fa-copy'
+ );
+ fileItem.duplicateWarning = warning;
+ }
- recordings.value.unshift(data);
- totalRecordings.value++;
- pollProcessingStatus(nextFileItem);
+ } catch (error) {
+ console.error(`Upload Error for ${fileItem.file.name} (Client ID: ${fileItem.clientId}):`, error);
+ fileItem.status = 'failed';
+ fileItem.error = error.message;
+ fileItem.progress = 0;
- } else {
- throw new Error('Unexpected success response from server after upload.');
- }
+ // Show friendly error message
+ const friendlyErr = getFriendlyError(error.message);
+ setGlobalError(`${friendlyErr.title}: ${friendlyErr.guidance}`);
- } catch (error) {
- console.error(`Upload/Processing Error for ${nextFileItem.file.name} (Client ID: ${nextFileItem.clientId}):`, error);
- nextFileItem.status = 'failed';
- nextFileItem.error = error.message;
- const failedRecordIndex = recordings.value.findIndex(r => r.id === nextFileItem.recordingId);
- if (failedRecordIndex !== -1) {
- recordings.value[failedRecordIndex].status = 'FAILED';
- recordings.value[failedRecordIndex].transcription = `Upload/Processing failed: ${error.message}`;
- } else {
- // Show friendly error message in toast
- const friendlyErr = getFriendlyError(error.message);
- setGlobalError(`${friendlyErr.title}: ${friendlyErr.guidance}`);
- }
+ // Store failed upload in IndexedDB for background sync retry
+ try {
+ await FailedUploads.storeFailedUpload({
+ file: fileItem.file,
+ fileName: fileItem.file.name,
+ fileSize: fileItem.file.size,
+ clientId: fileItem.clientId,
+ notes: fileItem.notes,
+ tags: fileItem.tags,
+ asrOptions: fileItem.asrOptions,
+ error: error.message
+ });
- // Store failed upload in IndexedDB for background sync retry
- try {
- await FailedUploads.storeFailedUpload({
- file: nextFileItem.file,
- fileName: nextFileItem.file.name,
- fileSize: nextFileItem.file.size,
- clientId: nextFileItem.clientId,
- notes: nextFileItem.notes,
- tags: nextFileItem.tags,
- asrOptions: nextFileItem.asrOptions,
- error: error.message
- });
-
- // Register for background sync
- if ('serviceWorker' in navigator && 'sync' in ServiceWorkerRegistration.prototype) {
- const registration = await navigator.serviceWorker.ready;
- await registration.sync.register('sync-uploads');
- console.log('[Upload] Registered background sync for failed upload');
- showToast('Upload will retry automatically when connection is restored', 'info');
- }
- } catch (syncError) {
- console.warn('[Upload] Failed to register background sync:', syncError);
+ if ('serviceWorker' in navigator && 'sync' in ServiceWorkerRegistration.prototype) {
+ const registration = await navigator.serviceWorker.ready;
+ await registration.sync.register('sync-uploads');
+ console.log('[Upload] Registered background sync for failed upload');
}
-
- resetCurrentFileProcessingState();
- isProcessingActive.value = false;
- await Vue.nextTick();
- startProcessingQueue();
+ } catch (syncError) {
+ console.warn('[Upload] Failed to register background sync:', syncError);
}
- } else {
- console.log("Upload queue is empty or no files are queued.");
- isProcessingActive.value = false;
+ } finally {
+ fileItem._xhr = null;
+ releaseUploadSlot();
}
};
- const startStatusPolling = (fileItem, recordingId) => {
- fileItem.recordingId = recordingId;
- pollProcessingStatus(fileItem);
- };
-
- const pollProcessingStatus = (fileItem) => {
- if (pollInterval.value) clearInterval(pollInterval.value);
-
- const recordingId = fileItem.recordingId;
- if (!recordingId) {
- console.error("Cannot poll status without recording ID for", fileItem.file.name);
- fileItem.status = 'failed';
- fileItem.error = 'Internal error: Missing recording ID for polling.';
- resetCurrentFileProcessingState();
- isProcessingActive.value = false;
- Vue.nextTick(startProcessingQueue);
+ /**
+ * Start uploading all ready files in parallel (with concurrency limit).
+ * Processing status is tracked via allJobs polling in app.modular.js.
+ */
+ const startProcessingQueue = async () => {
+ const readyItems = uploadQueue.value.filter(item => item.status === 'ready');
+ if (readyItems.length === 0) {
+ console.log("No files ready to upload.");
return;
}
- processingMessage.value = 'Waiting for transcription...';
- processingProgress.value = 40;
-
- pollInterval.value = setInterval(async () => {
- const shouldStopPolling = !currentlyProcessingFile.value ||
- currentlyProcessingFile.value.clientId !== fileItem.clientId ||
- fileItem.status === 'failed' ||
- (fileItem.status === 'completed' && (!fileItem.willAutoSummarize || fileItem.summaryCompleted));
-
- if (shouldStopPolling) {
- console.log(`Polling stopped for ${fileItem.clientId} as it's no longer active or finished.`);
- clearInterval(pollInterval.value);
- pollInterval.value = null;
- if (currentlyProcessingFile.value && currentlyProcessingFile.value.clientId === fileItem.clientId) {
- resetCurrentFileProcessingState();
- isProcessingActive.value = false;
- await Vue.nextTick();
- startProcessingQueue();
- }
- return;
- }
-
- try {
- console.log(`Polling status for recording ID: ${recordingId} (${fileItem.file.name})`);
- // Use lightweight status-only endpoint
- const response = await fetch(`/recording/${recordingId}/status`);
- if (!response.ok) throw new Error(`Status check failed with status ${response.status}`);
-
- // Safely parse JSON response, handling HTML error pages
- const statusContentType = response.headers.get('content-type') || '';
- if (!statusContentType.includes('application/json')) {
- const text = await response.text();
- const titleMatch = text.match(/<title>([^<]+)<\/title>/i);
- const h1Match = text.match(/<h1>([^<]+)<\/h1>/i);
- throw new Error(titleMatch?.[1] || h1Match?.[1] ||
- `Server error (${response.status}): Status response was not JSON`);
- }
- const statusData = await response.json();
- const galleryIndex = recordings.value.findIndex(r => r.id === recordingId);
-
- // Update status in recordings list
- if (galleryIndex !== -1) {
- // Create new object to ensure Vue reactivity
- recordings.value[galleryIndex] = {
- ...recordings.value[galleryIndex],
- status: statusData.status
- };
-
- // Update selectedRecording with new object reference for reactivity
- if (selectedRecording.value?.id === recordingId) {
- selectedRecording.value = {
- ...selectedRecording.value,
- status: statusData.status
- };
- }
- }
-
- const previousStatus = fileItem.status;
- fileItem.status = statusData.status;
-
- if (statusData.status === 'COMPLETED') {
- // Fetch full recording data when complete
- const fullResponse = await fetch(`/api/recordings/${recordingId}`);
- if (fullResponse.ok) {
- const data = await fullResponse.json();
-
- // Update recordings list first
- if (galleryIndex !== -1) {
- recordings.value[galleryIndex] = data;
- }
-
- // Always update selectedRecording if it's the current recording,
- // even if it's not in the current recordings list (e.g., filtered out)
- if (selectedRecording.value?.id === recordingId) {
- selectedRecording.value = data;
- // Force Vue to detect the change
- await nextTick();
- }
-
- // Store display name separately since File.name is read-only
- fileItem.displayName = data.title || data.original_filename || fileItem.file.name;
- }
-
- console.log(`Processing COMPLETED for ${fileItem.displayName} (ID: ${recordingId})`);
-
- if (previousStatus === 'summarizing') {
- console.log(`Auto-summary completed for ${fileItem.displayName}`);
- processingMessage.value = 'Processing complete!';
- processingProgress.value = 100;
- fileItem.status = 'completed';
- fileItem.summaryCompleted = true;
-
- clearInterval(pollInterval.value);
- pollInterval.value = null;
- resetCurrentFileProcessingState();
- isProcessingActive.value = false;
- // Refresh token budget after LLM operations
- if (onChatComplete) onChatComplete();
-
- startProcessingQueue();
- return;
- } else if (fileItem.willAutoSummarize && !fileItem.hasCheckedForAutoSummary) {
- processingMessage.value = 'Transcription complete!';
- processingProgress.value = 85;
- fileItem.status = 'awaiting_summary';
- fileItem.hasCheckedForAutoSummary = true;
- fileItem.autoSummaryStartTime = Date.now();
- return;
- } else if (fileItem.willAutoSummarize && fileItem.hasCheckedForAutoSummary) {
- const waitTime = Date.now() - fileItem.autoSummaryStartTime;
- const maxWaitTime = 5000; // 5 seconds - if no summarization starts, complete
-
- if (waitTime > maxWaitTime) {
- console.log(`Auto-summary did not start within ${maxWaitTime}ms, completing`);
- processingMessage.value = 'Processing complete!';
- processingProgress.value = 100;
- fileItem.status = 'completed';
- fileItem.summaryCompleted = false; // Summary didn't happen
- clearInterval(pollInterval.value);
- pollInterval.value = null;
- resetCurrentFileProcessingState();
- isProcessingActive.value = false;
- // Refresh token budget after LLM operations
- if (onChatComplete) onChatComplete();
- startProcessingQueue();
- return;
- }
- return;
- } else {
- processingMessage.value = 'Processing complete!';
- processingProgress.value = 100;
- fileItem.status = 'completed';
- fileItem.summaryCompleted = true;
-
- clearInterval(pollInterval.value);
- pollInterval.value = null;
- resetCurrentFileProcessingState();
- isProcessingActive.value = false;
- // Refresh token budget after LLM operations
- if (onChatComplete) onChatComplete();
- startProcessingQueue();
- return;
- }
-
- } else if (statusData.status === 'FAILED') {
- console.log(`Processing FAILED for ${fileItem.displayName || fileItem.file.name} (ID: ${recordingId})`);
- processingMessage.value = 'Processing failed.';
- processingProgress.value = 100;
- fileItem.status = 'failed';
+ isProcessingActive.value = true;
+ console.log(`Starting parallel upload of ${readyItems.length} file(s) (max ${MAX_CONCURRENT_UPLOADS} concurrent)...`);
+
+ // Fire off all uploads concurrently (semaphore handles limiting)
+ const uploadPromises = readyItems.map(item => uploadSingleFile(item));
+ // Don't await - let them run in background. isProcessingActive is cleared by releaseUploadSlot.
+ Promise.allSettled(uploadPromises).then(() => {
+ console.log('All uploads settled.');
+ });
+ };
- // Fetch full data to get error details and update UI
- try {
- const failedResponse = await fetch(`/api/recordings/${recordingId}`);
- if (failedResponse.ok) {
- const failedData = await failedResponse.json();
- fileItem.error = failedData.error_message || 'Processing failed on server.';
-
- // Update recordings list so error shows in detail view
- const galleryIndex = recordings.value.findIndex(r => r.id === recordingId);
- if (galleryIndex !== -1) {
- recordings.value[galleryIndex] = failedData;
- }
-
- // Update selectedRecording to show error in transcription panel
- if (selectedRecording.value?.id === recordingId) {
- selectedRecording.value = failedData;
- await nextTick();
- }
- } else {
- fileItem.error = 'Processing failed on server.';
- }
- } catch (err) {
- fileItem.error = 'Processing failed on server.';
- }
+ // Keep backward-compat aliases
+ const startStatusPolling = (fileItem, recordingId) => {
+ // No longer needed - allJobs polling handles status tracking
+ fileItem.recordingId = recordingId;
+ };
- // Show friendly error message in toast
- const friendlyErr = getFriendlyError(fileItem.error);
- setGlobalError(`${friendlyErr.title}: ${friendlyErr.guidance}`);
- clearInterval(pollInterval.value);
- pollInterval.value = null;
- resetCurrentFileProcessingState();
- isProcessingActive.value = false;
- await Vue.nextTick();
- startProcessingQueue();
-
- } else if (statusData.status === 'PROCESSING') {
- const couldUseChunking = chunkingEnabled.value && !useAsrEndpoint.value;
-
- if (couldUseChunking) {
- if (chunkingMode.value === 'size') {
- const chunkThresholdBytes = chunkingLimit.value * 1024 * 1024;
- const willUseChunking = fileItem.file.size > chunkThresholdBytes;
-
- if (willUseChunking) {
- processingMessage.value = 'Processing large file (chunking in progress)...';
- const maxProgress = fileItem.willAutoSummarize ? 70 : 80;
- processingProgress.value = Math.round(Math.min(maxProgress, processingProgress.value + Math.random() * 3));
- } else {
- processingMessage.value = 'Transcription in progress...';
- const maxProgress = fileItem.willAutoSummarize ? 65 : 75;
- processingProgress.value = Math.round(Math.min(maxProgress, processingProgress.value + Math.random() * 5));
- }
- } else {
- processingMessage.value = 'Processing file (chunking determined server-side)...';
- const maxProgress = fileItem.willAutoSummarize ? 70 : 80;
- processingProgress.value = Math.round(Math.min(maxProgress, processingProgress.value + Math.random() * 3));
- }
- } else {
- processingMessage.value = 'Transcription in progress...';
- const maxProgress = fileItem.willAutoSummarize ? 65 : 75;
- processingProgress.value = Math.round(Math.min(maxProgress, processingProgress.value + Math.random() * 5));
- }
- } else if (statusData.status === 'SUMMARIZING') {
- console.log(`Auto-summary started for ${fileItem.displayName || fileItem.file.name}`);
- processingMessage.value = 'Generating summary...';
- processingProgress.value = 90;
- fileItem.status = 'summarizing';
- } else {
- processingMessage.value = 'Waiting in queue...';
- processingProgress.value = 45;
- }
- } catch (error) {
- console.error(`Polling Error for ${fileItem.displayName || fileItem.file.name} (ID: ${recordingId}):`, error);
- fileItem.status = 'failed';
- fileItem.error = `Error checking status: ${error.message}`;
- setGlobalError(`Error checking status for "${fileItem.displayName || fileItem.file.name}": ${error.message}.`);
- const galleryIndex = recordings.value.findIndex(r => r.id === recordingId);
- if (galleryIndex !== -1) recordings.value[galleryIndex].status = 'FAILED';
-
- clearInterval(pollInterval.value);
- pollInterval.value = null;
- resetCurrentFileProcessingState();
- isProcessingActive.value = false;
- await Vue.nextTick();
- startProcessingQueue();
- }
- }, 5000);
+ const pollProcessingStatus = () => {
+ // No-op: status tracking is now handled by allJobs polling in app.modular.js
};
// Tag selection helpers
diff --git a/static/locales/de.json b/static/locales/de.json
index d5a5ec7d..d3d1bd06 100644
--- a/static/locales/de.json
+++ b/static/locales/de.json
@@ -1144,7 +1144,8 @@
"transcribing": "Transkribiere...",
"untitled": "Unbenannte Aufnahme",
"uploadProgress": "Upload-Fortschritt",
- "willAutoSummarize": "Wird nach der Transkription automatisch zusammengefasst"
+ "willAutoSummarize": "Wird nach der Transkription automatisch zusammengefasst",
+ "duplicateDetected": "Diese Datei scheint ein Duplikat von \"{{existingName}}\" zu sein (hochgeladen am {{existingDate}})"
},
"uploadProgress": {
"title": "Upload-Fortschritt"
diff --git a/static/locales/en.json b/static/locales/en.json
index dda9c25e..8ea7c22d 100644
--- a/static/locales/en.json
+++ b/static/locales/en.json
@@ -296,7 +296,6 @@
"currentUsageMinutes": "Current Usage",
"userTranscriptionUsage": "User Transcription Usage (This Month)",
"noTranscriptionData": "No transcription usage data available",
- "minutes": "minutes",
"percentUsed": "used",
"topUsers": "Top Users",
"topUsersByStorage": "Top Users by Storage",
@@ -403,20 +402,56 @@
"title": "Choose Color Scheme",
"themes": {
"light": {
- "blue": { "name": "Ocean Blue", "description": "Classic blue theme with professional appeal" },
- "emerald": { "name": "Forest Emerald", "description": "Fresh green theme for a natural feel" },
- "purple": { "name": "Royal Purple", "description": "Elegant purple theme with sophistication" },
- "rose": { "name": "Sunset Rose", "description": "Warm pink theme with gentle energy" },
- "amber": { "name": "Golden Amber", "description": "Warm yellow theme for brightness" },
- "teal": { "name": "Ocean Teal", "description": "Cool teal theme for tranquility" }
+ "blue": {
+ "name": "Ocean Blue",
+ "description": "Classic blue theme with professional appeal"
+ },
+ "emerald": {
+ "name": "Forest Emerald",
+ "description": "Fresh green theme for a natural feel"
+ },
+ "purple": {
+ "name": "Royal Purple",
+ "description": "Elegant purple theme with sophistication"
+ },
+ "rose": {
+ "name": "Sunset Rose",
+ "description": "Warm pink theme with gentle energy"
+ },
+ "amber": {
+ "name": "Golden Amber",
+ "description": "Warm yellow theme for brightness"
+ },
+ "teal": {
+ "name": "Ocean Teal",
+ "description": "Cool teal theme for tranquility"
+ }
},
"dark": {
- "blue": { "name": "Midnight Blue", "description": "Deep blue theme for focused work" },
- "emerald": { "name": "Dark Forest", "description": "Rich green theme for comfortable viewing" },
- "purple": { "name": "Deep Purple", "description": "Mysterious purple theme for creativity" },
- "rose": { "name": "Dark Rose", "description": "Muted pink theme with subtle warmth" },
- "amber": { "name": "Dark Amber", "description": "Warm brown theme for cozy sessions" },
- "teal": { "name": "Deep Teal", "description": "Dark teal theme for calm focus" }
+ "blue": {
+ "name": "Midnight Blue",
+ "description": "Deep blue theme for focused work"
+ },
+ "emerald": {
+ "name": "Dark Forest",
+ "description": "Rich green theme for comfortable viewing"
+ },
+ "purple": {
+ "name": "Deep Purple",
+ "description": "Mysterious purple theme for creativity"
+ },
+ "rose": {
+ "name": "Dark Rose",
+ "description": "Muted pink theme with subtle warmth"
+ },
+ "amber": {
+ "name": "Dark Amber",
+ "description": "Warm brown theme for cozy sessions"
+ },
+ "teal": {
+ "name": "Deep Teal",
+ "description": "Dark teal theme for calm focus"
+ }
}
}
},
@@ -703,7 +738,7 @@
"speakerRequirement": "Speaker identification requires recordings with multiple speakers",
"activeFilters": "Active Filters:",
"tagsCount": "tags",
- "speakersCount": "speakers",
+ "speakersCount": "speakers",
"dateRangeActive": "Date range active",
"filtersActive": "Filters active",
"askQuestions": "Ask Questions About Your Transcriptions",
@@ -964,7 +999,6 @@
"searchRecordings": "Search recordings...",
"sharedWithMe": "Shared with Me",
"starred": "Starred",
- "inbox": "Inbox",
"sortBy": "Sort By",
"sortByDate": "Created Date",
"sortByMeetingDate": "Meeting Date",
@@ -1161,7 +1195,8 @@
"transcribing": "Transcribing...",
"untitled": "Untitled Recording",
"uploadProgress": "Upload Progress",
- "willAutoSummarize": "Will auto-summarize after transcription"
+ "willAutoSummarize": "Will auto-summarize after transcription",
+ "duplicateDetected": "This file appears to be a duplicate of \"{{existingName}}\" (uploaded {{existingDate}})"
},
"uploadProgress": {
"title": "Upload Progress"
@@ -1270,4 +1305,4 @@
"recordingData": "Recording Data",
"contentSections": "Content Sections"
}
-}
\ No newline at end of file
+}
diff --git a/static/locales/es.json b/static/locales/es.json
index 5e20c9c0..2977513c 100644
--- a/static/locales/es.json
+++ b/static/locales/es.json
@@ -1144,7 +1144,8 @@
"transcribing": "Transcribiendo...",
"untitled": "Grabación Sin Título",
"uploadProgress": "Progreso de Carga",
- "willAutoSummarize": "Se resumirá automáticamente después de la transcripción"
+ "willAutoSummarize": "Se resumirá automáticamente después de la transcripción",
+ "duplicateDetected": "Este archivo parece ser un duplicado de \"{{existingName}}\" (subido el {{existingDate}})"
},
"uploadProgress": {
"title": "Progreso de Carga"
diff --git a/static/locales/fr.json b/static/locales/fr.json
index f137351e..fc885340 100644
--- a/static/locales/fr.json
+++ b/static/locales/fr.json
@@ -1138,7 +1138,8 @@
"transcribing": "Transcription...",
"untitled": "Enregistrement Sans Titre",
"uploadProgress": "Progression du Téléchargement",
- "willAutoSummarize": "Résumé automatique après la transcription"
+ "willAutoSummarize": "Résumé automatique après la transcription",
+ "duplicateDetected": "Ce fichier semble être un doublon de \"{{existingName}}\" (importé le {{existingDate}})"
},
"uploadProgress": {
"title": "Progression du Téléchargement"
diff --git a/static/locales/ru.json b/static/locales/ru.json
index ae717ea9..57e1996c 100644
--- a/static/locales/ru.json
+++ b/static/locales/ru.json
@@ -1160,7 +1160,8 @@
"transcribing": "Транскрибирование...",
"untitled": "Запись без названия",
"uploadProgress": "Прогресс загрузки",
- "willAutoSummarize": "Автоматически создаст резюме после транскрибирования"
+ "willAutoSummarize": "Автоматически создаст резюме после транскрибирования",
+ "duplicateDetected": "Этот файл является дубликатом \"{{existingName}}\" (загружен {{existingDate}})"
},
"uploadProgress": {
"title": "Прогресс загрузки"
diff --git a/static/locales/zh.json b/static/locales/zh.json
index c32ca8a8..b2b13e20 100644
--- a/static/locales/zh.json
+++ b/static/locales/zh.json
@@ -1273,7 +1273,8 @@
"transcribing": "转录中...",
"untitled": "无标题录音",
"uploadProgress": "上传进度",
- "willAutoSummarize": "转录后将自动生成摘要"
+ "willAutoSummarize": "转录后将自动生成摘要",
+ "duplicateDetected": "此文件似乎与 \"{{existingName}}\" 重复(上传于 {{existingDate}})"
},
"uploadProgress": {
"title": "上传进度"
diff --git a/templates/components/progress-popup.html b/templates/components/progress-popup.html
index e38c9c9e..f8a68795 100644
--- a/templates/components/progress-popup.html
+++ b/templates/components/progress-popup.html
@@ -173,6 +173,7 @@
class="px-2 py-1 bg-[var(--bg-tertiary)]/30 rounded-md flex items-center gap-2 hover:bg-[var(--bg-tertiary)]/50 cursor-pointer mb-1">
<i class="fas fa-check-circle text-xs text-green-500"></i>
<span class="text-xs truncate flex-1 text-[var(--text-primary)]">${item.title}</span>
+ <i v-if="item.duplicateWarning" class="fas fa-copy text-xs text-amber-500" title="Duplicate file detected"></i>
<span class="text-[10px] px-1.5 py-0.5 rounded bg-green-500/20 text-green-500 font-medium">Done</span>
</div>
</div>