Speakr is a personal, self-hosted web application designed for transcribing audio recordings
Python
feat: segment-aware chunking (whole speaker turns, dominant-speaker + timestamp metadata, CHUNKING_STRATEGY=legacy escape hatch), embedding API batching, timestamped Inquire citations with ?t= playback deep links
Murtaza Nasir5e5dbf0d9711
diff --git a/config/env.asr.example b/config/env.asr.exampleindex 2004e98c..5f7e8207 100644--- a/config/env.asr.example+++ b/config/env.asr.example@@ -234,6 +234,20 @@ ENABLE_INQUIRE_MODE=false# set the endpoint and API key below. When EMBEDDING_BASE_URL is set,# Inquire embeddings are generated via the API instead of locally.# EMBEDDING_BASE_URL=+#+# Chunking for search indexing. Diarized transcripts are chunked+# segment-aware ("Speaker: text" lines, whole turns kept together, speaker+# and timestamps stored per chunk). Set CHUNKING_STRATEGY=legacy to keep the+# old plain 500-character windows instead. Existing chunks are unaffected+# either way until a recording is reindexed.+# CHUNKING_STRATEGY=segment+# CHUNK_TARGET_CHARS=1400+#+# Embedding API robustness: inputs per request (providers cap this), and how+# long to skip a failing embeddings endpoint before probing again (searches+# fall back to keyword matching in the meantime).+# EMBEDDING_API_BATCH_SIZE=96+# EMBEDDING_API_COOLDOWN_SECONDS=120# EMBEDDING_API_KEY=# Optional fixed embedding dimensionality. Leave empty to auto-detect fromdiff --git a/config/env.transcription.example b/config/env.transcription.exampleindex f55c767b..cf2ed5be 100644--- a/config/env.transcription.example+++ b/config/env.transcription.example@@ -403,6 +403,20 @@ ENABLE_INQUIRE_MODE=false# set the endpoint and API key below. When EMBEDDING_BASE_URL is set,# Inquire embeddings are generated via the API instead of locally.# EMBEDDING_BASE_URL=+#+# Chunking for search indexing. Diarized transcripts are chunked+# segment-aware ("Speaker: text" lines, whole turns kept together, speaker+# and timestamps stored per chunk). Set CHUNKING_STRATEGY=legacy to keep the+# old plain 500-character windows instead. Existing chunks are unaffected+# either way until a recording is reindexed.+# CHUNKING_STRATEGY=segment+# CHUNK_TARGET_CHARS=1400+#+# Embedding API robustness: inputs per request (providers cap this), and how+# long to skip a failing embeddings endpoint before probing again (searches+# fall back to keyword matching in the meantime).+# EMBEDDING_API_BATCH_SIZE=96+# EMBEDDING_API_COOLDOWN_SECONDS=120# EMBEDDING_API_KEY=# Optional fixed embedding dimensionality. Leave empty to auto-detect fromdiff --git a/config/env.whisper.example b/config/env.whisper.exampleindex 379cb30c..d63d1aa2 100644--- a/config/env.whisper.example+++ b/config/env.whisper.example@@ -230,6 +230,20 @@ ENABLE_INQUIRE_MODE=false# set the endpoint and API key below. When EMBEDDING_BASE_URL is set,# Inquire embeddings are generated via the API instead of locally.# EMBEDDING_BASE_URL=+#+# Chunking for search indexing. Diarized transcripts are chunked+# segment-aware ("Speaker: text" lines, whole turns kept together, speaker+# and timestamps stored per chunk). Set CHUNKING_STRATEGY=legacy to keep the+# old plain 500-character windows instead. Existing chunks are unaffected+# either way until a recording is reindexed.+# CHUNKING_STRATEGY=segment+# CHUNK_TARGET_CHARS=1400+#+# Embedding API robustness: inputs per request (providers cap this), and how+# long to skip a failing embeddings endpoint before probing again (searches+# fall back to keyword matching in the meantime).+# EMBEDDING_API_BATCH_SIZE=96+# EMBEDDING_API_COOLDOWN_SECONDS=120# EMBEDDING_API_KEY=# Optional fixed embedding dimensionality. Leave empty to auto-detect fromdiff --git a/src/services/embeddings.py b/src/services/embeddings.pyindex 840dc46f..255861f8 100644--- a/src/services/embeddings.py+++ b/src/services/embeddings.py@@ -4,6 +4,7 @@ Embedding generation and semantic search services.import osimport re+import jsonimport timeimport randomimport numpy as np@@ -178,6 +179,20 @@ def _api_embed(texts, user_id=None):if client is None or not texts:return []+ # Batch large inputs: providers cap the number of inputs per embeddings+ # request (Gemini far lower than OpenAI), and a long recording can have+ # hundreds of chunks. All-or-nothing semantics are preserved: any failed+ # batch fails the whole call so callers keep their existing chunks.+ batch_size = max(1, int(os.environ.get('EMBEDDING_API_BATCH_SIZE', '96')))+ if len(texts) > batch_size:+ out = []+ for i in range(0, len(texts), batch_size):+ part = _api_embed(texts[i:i + batch_size], user_id=user_id)+ if len(part) != len(texts[i:i + batch_size]):+ return []+ out.extend(part)+ return out+# Circuit breaker: after a full retry cycle fails, skip the API entirely# for a cooldown window instead of burning ~9s of retries on every call# (searches fall back to keyword matching immediately; indexing fails@@ -261,6 +276,126 @@ def _api_embed(texts, user_id=None):+# Segment-aware chunking target. ~1400 chars ≈ 300-350 tokens: large enough+# that a rapid back-and-forth exchange stays together in one chunk, small+# enough to stay a precise retrieval unit.+CHUNK_TARGET_CHARS = int(os.environ.get('CHUNK_TARGET_CHARS', '1400'))+++def build_transcript_segments(transcription):+ """Parse a stored transcription into [{speaker, text, start, end}].++ Returns None when the transcription is not the JSON segment format+ (plain-text transcripts fall back to character windowing).+ """+ if not transcription:+ return None+ try:+ data = json.loads(transcription)+ except (ValueError, TypeError):+ return None+ if not isinstance(data, list):+ return None+ segments = []+ for seg in data:+ if not isinstance(seg, dict):+ continue+ text = (seg.get('sentence') or seg.get('text') or '').strip()+ if not text:+ continue+ segments.append({+ 'speaker': (seg.get('speaker') or '').strip() or None,+ 'text': text,+ 'start': seg.get('start_time'),+ 'end': seg.get('end_time'),+ })+ return segments if segments else None+++def _segment_line(seg):+ """Render one segment as a transcript-style line: 'Speaker: text'."""+ return f"{seg['speaker']}: {seg['text']}" if seg['speaker'] else seg['text']+++def _finalize_segment_chunk(segs):+ """Build a chunk dict from a run of consecutive segments."""+ text = "\n".join(_segment_line(s) for s in segs)+ speakers = [s['speaker'] for s in segs if s['speaker']]+ # Dominant speaker goes in the metadata column (single exact name, so the+ # speaker-rename maintenance which matches by equality keeps working);+ # every speaker remains visible inline in the chunk text itself.+ dominant = max(set(speakers), key=speakers.count) if speakers else None+ starts = [s['start'] for s in segs if s['start'] is not None]+ ends = [s['end'] for s in segs if s['end'] is not None]+ return {+ 'text': text,+ 'speaker_name': dominant,+ 'start_time': min(starts) if starts else None,+ 'end_time': max(ends) if ends else None,+ }+++def chunk_transcript_segments(transcription, max_chunk_chars=None):+ """Segment-aware chunking for diarized (JSON) transcripts.++ Strategy: render each segment as a 'Speaker: text' line and pack WHOLE+ segments into chunks up to ``max_chunk_chars``, never splitting a+ speaker turn across chunks (except single turns longer than the limit,+ which are sentence-split on their own). Chunk boundaries are size-based,+ not turn-based, so a quickfire exchange of many short turns lands+ together in one chunk with every speaker labeled inline. Consecutive+ chunks overlap by one segment for continuity.++ Returns a list of chunk dicts: {text, speaker_name (dominant),+ start_time, end_time}. Plain-text transcripts fall back to the classic+ character windows with null metadata.+ """+ # Backwards-compatibility escape hatch: CHUNKING_STRATEGY=legacy keeps+ # the pre-segment-aware behavior byte-identical (500-char windows over+ # the stored string, no speaker/time metadata) for admins who do not+ # want indexing behavior to change. Existing chunks are never touched+ # by an upgrade either way; the strategy only applies when a recording+ # is (re)indexed.+ if os.environ.get('CHUNKING_STRATEGY', 'segment').strip().lower() == 'legacy':+ return [{'text': t, 'speaker_name': None, 'start_time': None, 'end_time': None}+ for t in chunk_transcription(transcription)]++ max_chunk_chars = max_chunk_chars or CHUNK_TARGET_CHARS+ segments = build_transcript_segments(transcription)+ if segments is None:+ return [{'text': t, 'speaker_name': None, 'start_time': None, 'end_time': None}+ for t in chunk_transcription(transcription, max_chunk_length=max_chunk_chars,+ overlap=max(50, max_chunk_chars // 20))]++ chunks = []+ current = []+ current_len = 0+ for seg in segments:+ line_len = len(_segment_line(seg)) + 1+ if line_len > max_chunk_chars:+ # A single oversized turn: flush what we have, then sentence-split+ # the turn into its own chunks carrying its speaker and times.+ if current:+ chunks.append(_finalize_segment_chunk(current))+ current, current_len = [], 0+ for piece in chunk_transcription(seg['text'], max_chunk_length=max_chunk_chars,+ overlap=max(50, max_chunk_chars // 20)):+ chunks.append(_finalize_segment_chunk([{**seg, 'text': piece}]))+ continue+ if current and current_len + line_len > max_chunk_chars:+ chunks.append(_finalize_segment_chunk(current))+ # One-segment overlap so a thought spanning the boundary stays+ # findable from either side.+ tail = current[-1]+ current = [tail] if len(_segment_line(tail)) + line_len <= max_chunk_chars else []+ current_len = sum(len(_segment_line(s)) + 1 for s in current)+ current.append(seg)+ current_len += line_len+ if current:+ chunks.append(_finalize_segment_chunk(current))+ return chunks++def chunk_transcription(transcription, max_chunk_length=500, overlap=50):"""Split transcription into overlapping chunks for better context retrieval.@@ -420,7 +555,7 @@ def process_recording_chunks(recording_id):# for the whole duration and starved concurrent writers such as the# summarize-job enqueue (issue #355). Doing the slow work first keeps# the delete + insert + commit window down to milliseconds.- chunks = chunk_transcription(recording.transcription)+ chunks = chunk_transcript_segments(recording.transcription)if not chunks:TranscriptChunk.query.filter_by(recording_id=recording_id).delete()@@ -428,7 +563,7 @@ def process_recording_chunks(recording_id):return True# Generate embeddings (recording owner gets billed for API-mode usage)- embeddings = generate_embeddings(chunks, user_id=recording.user_id)+ embeddings = generate_embeddings([c['text'] for c in chunks], user_id=recording.user_id)# Verify we got one embedding per chunk. _api_embed returns [] on# exhausted retries, and a partial provider response could return@@ -447,12 +582,15 @@ def process_recording_chunks(recording_id):# only the final commit makes it permanent, so a failure here still# rolls back to the old chunks.TranscriptChunk.query.filter_by(recording_id=recording_id).delete()- for i, (chunk_text, embedding) in enumerate(zip(chunks, embeddings)):+ for i, (chunk_data, embedding) in enumerate(zip(chunks, embeddings)):chunk = TranscriptChunk(recording_id=recording_id,user_id=recording.user_id,chunk_index=i,- content=chunk_text,+ content=chunk_data['text'],+ speaker_name=(chunk_data['speaker_name'] or None) and chunk_data['speaker_name'][:100],+ start_time=chunk_data['start_time'],+ end_time=chunk_data['end_time'],embedding=serialize_embedding(embedding) if embedding is not None else None)db.session.add(chunk)diff --git a/src/services/inquire_agent.py b/src/services/inquire_agent.pyindex c91e4148..4ca23577 100644--- a/src/services/inquire_agent.py+++ b/src/services/inquire_agent.py@@ -279,6 +279,7 @@ def tool_search_transcripts(ctx, args):'meeting_date': chunk.recording.meeting_date.strftime('%Y-%m-%d') if chunk.recording.meeting_date else None,'speaker': chunk.speaker_name,'time': _fmt_time(chunk.start_time),+ 'seek_seconds': int(chunk.start_time) if chunk.start_time is not None else None,'chunk_index': chunk.chunk_index,'snippet': chunk.content,'similarity': round(float(similarity), 3),@@ -565,8 +566,11 @@ Never invent content that is not in the tool results; say clearly when something{tool_mode_prompt_block}When you give your final answer:- Use clear markdown (headings per recording where it helps, bullets, **bold** speaker names).-- Cite sources inline as markdown links: [Recording Title](/recordings/<recording_id>) — cite the \-recording you drew each substantive point from, at most one citation per bullet or paragraph.+- Cite sources inline as markdown links. When the supporting search result has a time and \+seek_seconds, link directly to that moment: [Title @ m:ss](/recordings/<recording_id>?t=<seek_seconds>) \+— for example [Weekly Sync @ 12:34](/recordings/73?t=754). Without a timestamp, link the recording: \+[Title](/recordings/<recording_id>). Cite the recording each substantive point came from, at most \+one citation per bullet or paragraph. These links open the recording and start playback at that moment.- Order information from the most recent recordings first unless the question implies otherwise."""diff --git a/static/js/app.modular.js b/static/js/app.modular.jsindex cf55e7d4..bed4f78e 100644--- a/static/js/app.modular.js+++ b/static/js/app.modular.js@@ -3877,11 +3877,39 @@ document.addEventListener('DOMContentLoaded', async () => {// Deep link to a specific recording: /recordings/<id> (#301).// Captured before load; selected once recordings are in.let deepLinkRecordingId = null;+ // Optional ?t=<seconds> seek (Inquire citations link to the+ // cited moment): after the recording opens, seek the main+ // player there and start playback.+ let deepLinkSeekSeconds = null;try {const m = window.location.pathname.match(/^\/recordings\/(\d+)\/?$/);if (m) deepLinkRecordingId = m[1];+ const t = new URLSearchParams(window.location.search).get('t');+ if (m && t !== null && /^\d+(\.\d+)?$/.test(t)) deepLinkSeekSeconds = parseFloat(t);} catch (_) { /* best-effort */ }+ const applyDeepLinkSeek = (seconds) => {+ const startedAt = Date.now();+ const attempt = () => {+ // The detail view's player is the media element whose+ // source is the recording's /audio/ stream.+ const media = [...document.querySelectorAll('audio, video')]+ .find(el => ((el.currentSrc || el.src || '').includes('/audio/')));+ if (media && media.readyState >= 1) {+ const target = isFinite(media.duration)+ ? Math.min(seconds, Math.max(0, media.duration - 1))+ : seconds;+ media.currentTime = target;+ // Best-effort autoplay; stays paused at the right+ // spot if the browser blocks it.+ media.play().catch(() => {});+ return;+ }+ if (Date.now() - startedAt < 15000) setTimeout(attempt, 400);+ };+ setTimeout(attempt, 400);+ };+// Load initial dataawait Promise.all([recordingsComposable.loadRecordings(),@@ -3900,6 +3928,9 @@ document.addEventListener('DOMContentLoaded', async () => {} else if (deepLinkRecordingId) {// Honour the /recordings/<id> deep link now that data is loaded.await recordingsComposable.selectRecordingById(deepLinkRecordingId);+ if (deepLinkSeekSeconds !== null) {+ applyDeepLinkSeek(deepLinkSeekSeconds);+ }}// Auto-retry any uploads that previously failed and were saved todiff --git a/tests/test_inquire_agent.py b/tests/test_inquire_agent.pyindex 18a13b3c..700d4bb9 100644--- a/tests/test_inquire_agent.py+++ b/tests/test_inquire_agent.py@@ -95,7 +95,8 @@ def event_types(events):USER_CTX = {'name': 'Testy', 'title': 'analyst', 'company': 'ACME', 'output_language': None}-SEGMENTS = [{'speaker': f'S{i % 2}', 'sentence': f'sentence number {i}', 'start_time': i * 2.0}+SEGMENTS = [{'speaker': f'S{i % 2}', 'sentence': f'sentence number {i}',+ 'start_time': i * 2.0, 'end_time': i * 2.0 + 1.5}for i in range(300)]@@ -371,6 +372,101 @@ def test_truncation_marker():assert 'TRUNCATED' in out and 'More available.' in out+# ---------------------------------------------------------------------------+# Segment-aware chunking+# ---------------------------------------------------------------------------++def _seg(speaker, text, start, end):+ return {'speaker': speaker, 'sentence': text, 'start_time': start, 'end_time': end}+++def test_chunking_quickfire_exchange_stays_together():+ """Many short turns pack into ONE chunk with every speaker labeled inline;+ chunk boundaries are size-based, never per-turn."""+ from src.services.embeddings import chunk_transcript_segments+ segs = []+ t = 0.0+ for i in range(20):+ who = 'Alice' if i % 2 == 0 else 'Bob'+ segs.append(_seg(who, f'quick reply {i}', t, t + 1.5))+ t += 2.0+ chunks = chunk_transcript_segments(json.dumps(segs), max_chunk_chars=1400)+ assert len(chunks) == 1+ c = chunks[0]+ assert c['text'].count('Alice:') == 10 and c['text'].count('Bob:') == 10+ assert c['start_time'] == 0.0 and c['end_time'] == pytest.approx(39.5)+ assert c['speaker_name'] in ('Alice', 'Bob')+++def test_chunking_dominant_speaker_and_times():+ from src.services.embeddings import chunk_transcript_segments+ segs = [+ _seg('Alice', 'point one', 10.0, 12.0),+ _seg('Alice', 'point two', 12.0, 14.0),+ _seg('Bob', 'brief interjection', 14.0, 15.0),+ ]+ chunks = chunk_transcript_segments(json.dumps(segs), max_chunk_chars=1400)+ assert len(chunks) == 1+ assert chunks[0]['speaker_name'] == 'Alice' # dominant+ assert chunks[0]['start_time'] == 10.0 and chunks[0]['end_time'] == 15.0+ assert 'Bob: brief interjection' in chunks[0]['text']+++def test_chunking_splits_on_size_with_segment_overlap():+ from src.services.embeddings import chunk_transcript_segments+ segs = [_seg('S1' if i % 2 == 0 else 'S2', f'sentence {i} ' + 'word ' * 20, i * 10.0, i * 10.0 + 8)+ for i in range(12)]+ chunks = chunk_transcript_segments(json.dumps(segs), max_chunk_chars=500)+ assert len(chunks) > 1+ # No chunk exceeds the cap; whole turns are never split across chunks+ for c in chunks:+ assert len(c['text']) <= 500 + 50+ for line in c['text'].split('\n'):+ assert line.startswith(('S1: ', 'S2: '))+ # One-segment overlap: last line of chunk N reappears as first line of N+1+ for a, b in zip(chunks, chunks[1:]):+ assert a['text'].split('\n')[-1] == b['text'].split('\n')[0]+ # Times track the segments actually in the chunk+ assert chunks[0]['start_time'] == 0.0+ assert chunks[-1]['end_time'] == segs[-1]['end_time']+++def test_chunking_oversized_single_turn_is_sentence_split():+ from src.services.embeddings import chunk_transcript_segments+ monologue = ('This is a long sentence about the quarterly results. ' * 30).strip()+ segs = [_seg('Speaker', monologue, 5.0, 300.0)]+ chunks = chunk_transcript_segments(json.dumps(segs), max_chunk_chars=400)+ assert len(chunks) > 1+ for c in chunks:+ assert c['speaker_name'] == 'Speaker'+ assert c['start_time'] == 5.0 and c['end_time'] == 300.0+++def test_chunking_plain_text_fallback_has_null_metadata():+ from src.services.embeddings import chunk_transcript_segments+ chunks = chunk_transcript_segments('just a plain text transcript. ' * 100, max_chunk_chars=500)+ assert len(chunks) > 1+ assert all(c['speaker_name'] is None and c['start_time'] is None for c in chunks)+++def test_process_recording_chunks_populates_metadata(data, monkeypatch):+ import numpy as np+ from src.services import embeddings as emb+ with app.app_context():+ monkeypatch.setattr(emb, 'generate_embeddings',+ lambda texts, user_id=None: [np.zeros(8, dtype=np.float32) for _ in texts])+ assert emb.process_recording_chunks(data['r1']) is True+ from src.models import TranscriptChunk+ chunks = TranscriptChunk.query.filter_by(recording_id=data['r1']).order_by(TranscriptChunk.chunk_index).all()+ assert chunks+ assert chunks[0].speaker_name in ('S0', 'S1')+ assert chunks[0].start_time == 0.0+ assert chunks[0].end_time is not None+ assert 'S0: sentence number 0' in chunks[0].content+ TranscriptChunk.query.filter_by(recording_id=data['r1']).delete()+ db.session.commit()++# ---------------------------------------------------------------------------# History and compaction# ---------------------------------------------------------------------------