Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python

feat: segment-aware chunking (whole speaker turns, dominant-speaker + timestamp metadata, CHUNKING_STRATEGY=legacy escape hatch), embedding API batching, timestamped Inquire citations with ?t= playback deep links

Murtaza Nasir5e5dbf0d9711
diff --git a/config/env.asr.example b/config/env.asr.example
index 2004e98c..5f7e8207 100644
--- a/config/env.asr.example
+++ b/config/env.asr.example
@@ -234,6 +234,20 @@ ENABLE_INQUIRE_MODE=false
# set the endpoint and API key below. When EMBEDDING_BASE_URL is set,
# Inquire embeddings are generated via the API instead of locally.
# EMBEDDING_BASE_URL=
+#
+# Chunking for search indexing. Diarized transcripts are chunked
+# segment-aware ("Speaker: text" lines, whole turns kept together, speaker
+# and timestamps stored per chunk). Set CHUNKING_STRATEGY=legacy to keep the
+# old plain 500-character windows instead. Existing chunks are unaffected
+# either way until a recording is reindexed.
+# CHUNKING_STRATEGY=segment
+# CHUNK_TARGET_CHARS=1400
+#
+# Embedding API robustness: inputs per request (providers cap this), and how
+# long to skip a failing embeddings endpoint before probing again (searches
+# fall back to keyword matching in the meantime).
+# EMBEDDING_API_BATCH_SIZE=96
+# EMBEDDING_API_COOLDOWN_SECONDS=120
# EMBEDDING_API_KEY=
# Optional fixed embedding dimensionality. Leave empty to auto-detect from
diff --git a/config/env.transcription.example b/config/env.transcription.example
index f55c767b..cf2ed5be 100644
--- a/config/env.transcription.example
+++ b/config/env.transcription.example
@@ -403,6 +403,20 @@ ENABLE_INQUIRE_MODE=false
# set the endpoint and API key below. When EMBEDDING_BASE_URL is set,
# Inquire embeddings are generated via the API instead of locally.
# EMBEDDING_BASE_URL=
+#
+# Chunking for search indexing. Diarized transcripts are chunked
+# segment-aware ("Speaker: text" lines, whole turns kept together, speaker
+# and timestamps stored per chunk). Set CHUNKING_STRATEGY=legacy to keep the
+# old plain 500-character windows instead. Existing chunks are unaffected
+# either way until a recording is reindexed.
+# CHUNKING_STRATEGY=segment
+# CHUNK_TARGET_CHARS=1400
+#
+# Embedding API robustness: inputs per request (providers cap this), and how
+# long to skip a failing embeddings endpoint before probing again (searches
+# fall back to keyword matching in the meantime).
+# EMBEDDING_API_BATCH_SIZE=96
+# EMBEDDING_API_COOLDOWN_SECONDS=120
# EMBEDDING_API_KEY=
# Optional fixed embedding dimensionality. Leave empty to auto-detect from
diff --git a/config/env.whisper.example b/config/env.whisper.example
index 379cb30c..d63d1aa2 100644
--- a/config/env.whisper.example
+++ b/config/env.whisper.example
@@ -230,6 +230,20 @@ ENABLE_INQUIRE_MODE=false
# set the endpoint and API key below. When EMBEDDING_BASE_URL is set,
# Inquire embeddings are generated via the API instead of locally.
# EMBEDDING_BASE_URL=
+#
+# Chunking for search indexing. Diarized transcripts are chunked
+# segment-aware ("Speaker: text" lines, whole turns kept together, speaker
+# and timestamps stored per chunk). Set CHUNKING_STRATEGY=legacy to keep the
+# old plain 500-character windows instead. Existing chunks are unaffected
+# either way until a recording is reindexed.
+# CHUNKING_STRATEGY=segment
+# CHUNK_TARGET_CHARS=1400
+#
+# Embedding API robustness: inputs per request (providers cap this), and how
+# long to skip a failing embeddings endpoint before probing again (searches
+# fall back to keyword matching in the meantime).
+# EMBEDDING_API_BATCH_SIZE=96
+# EMBEDDING_API_COOLDOWN_SECONDS=120
# EMBEDDING_API_KEY=
# Optional fixed embedding dimensionality. Leave empty to auto-detect from
diff --git a/src/services/embeddings.py b/src/services/embeddings.py
index 840dc46f..255861f8 100644
--- a/src/services/embeddings.py
+++ b/src/services/embeddings.py
@@ -4,6 +4,7 @@ Embedding generation and semantic search services.
import os
import re
+import json
import time
import random
import numpy as np
@@ -178,6 +179,20 @@ def _api_embed(texts, user_id=None):
if client is None or not texts:
return []
+ # Batch large inputs: providers cap the number of inputs per embeddings
+ # request (Gemini far lower than OpenAI), and a long recording can have
+ # hundreds of chunks. All-or-nothing semantics are preserved: any failed
+ # batch fails the whole call so callers keep their existing chunks.
+ batch_size = max(1, int(os.environ.get('EMBEDDING_API_BATCH_SIZE', '96')))
+ if len(texts) > batch_size:
+ out = []
+ for i in range(0, len(texts), batch_size):
+ part = _api_embed(texts[i:i + batch_size], user_id=user_id)
+ if len(part) != len(texts[i:i + batch_size]):
+ return []
+ out.extend(part)
+ return out
+
# Circuit breaker: after a full retry cycle fails, skip the API entirely
# for a cooldown window instead of burning ~9s of retries on every call
# (searches fall back to keyword matching immediately; indexing fails
@@ -261,6 +276,126 @@ def _api_embed(texts, user_id=None):
+# Segment-aware chunking target. ~1400 chars ≈ 300-350 tokens: large enough
+# that a rapid back-and-forth exchange stays together in one chunk, small
+# enough to stay a precise retrieval unit.
+CHUNK_TARGET_CHARS = int(os.environ.get('CHUNK_TARGET_CHARS', '1400'))
+
+
+def build_transcript_segments(transcription):
+ """Parse a stored transcription into [{speaker, text, start, end}].
+
+ Returns None when the transcription is not the JSON segment format
+ (plain-text transcripts fall back to character windowing).
+ """
+ if not transcription:
+ return None
+ try:
+ data = json.loads(transcription)
+ except (ValueError, TypeError):
+ return None
+ if not isinstance(data, list):
+ return None
+ segments = []
+ for seg in data:
+ if not isinstance(seg, dict):
+ continue
+ text = (seg.get('sentence') or seg.get('text') or '').strip()
+ if not text:
+ continue
+ segments.append({
+ 'speaker': (seg.get('speaker') or '').strip() or None,
+ 'text': text,
+ 'start': seg.get('start_time'),
+ 'end': seg.get('end_time'),
+ })
+ return segments if segments else None
+
+
+def _segment_line(seg):
+ """Render one segment as a transcript-style line: 'Speaker: text'."""
+ return f"{seg['speaker']}: {seg['text']}" if seg['speaker'] else seg['text']
+
+
+def _finalize_segment_chunk(segs):
+ """Build a chunk dict from a run of consecutive segments."""
+ text = "\n".join(_segment_line(s) for s in segs)
+ speakers = [s['speaker'] for s in segs if s['speaker']]
+ # Dominant speaker goes in the metadata column (single exact name, so the
+ # speaker-rename maintenance which matches by equality keeps working);
+ # every speaker remains visible inline in the chunk text itself.
+ dominant = max(set(speakers), key=speakers.count) if speakers else None
+ starts = [s['start'] for s in segs if s['start'] is not None]
+ ends = [s['end'] for s in segs if s['end'] is not None]
+ return {
+ 'text': text,
+ 'speaker_name': dominant,
+ 'start_time': min(starts) if starts else None,
+ 'end_time': max(ends) if ends else None,
+ }
+
+
+def chunk_transcript_segments(transcription, max_chunk_chars=None):
+ """Segment-aware chunking for diarized (JSON) transcripts.
+
+ Strategy: render each segment as a 'Speaker: text' line and pack WHOLE
+ segments into chunks up to ``max_chunk_chars``, never splitting a
+ speaker turn across chunks (except single turns longer than the limit,
+ which are sentence-split on their own). Chunk boundaries are size-based,
+ not turn-based, so a quickfire exchange of many short turns lands
+ together in one chunk with every speaker labeled inline. Consecutive
+ chunks overlap by one segment for continuity.
+
+ Returns a list of chunk dicts: {text, speaker_name (dominant),
+ start_time, end_time}. Plain-text transcripts fall back to the classic
+ character windows with null metadata.
+ """
+ # Backwards-compatibility escape hatch: CHUNKING_STRATEGY=legacy keeps
+ # the pre-segment-aware behavior byte-identical (500-char windows over
+ # the stored string, no speaker/time metadata) for admins who do not
+ # want indexing behavior to change. Existing chunks are never touched
+ # by an upgrade either way; the strategy only applies when a recording
+ # is (re)indexed.
+ if os.environ.get('CHUNKING_STRATEGY', 'segment').strip().lower() == 'legacy':
+ return [{'text': t, 'speaker_name': None, 'start_time': None, 'end_time': None}
+ for t in chunk_transcription(transcription)]
+
+ max_chunk_chars = max_chunk_chars or CHUNK_TARGET_CHARS
+ segments = build_transcript_segments(transcription)
+ if segments is None:
+ return [{'text': t, 'speaker_name': None, 'start_time': None, 'end_time': None}
+ for t in chunk_transcription(transcription, max_chunk_length=max_chunk_chars,
+ overlap=max(50, max_chunk_chars // 20))]
+
+ chunks = []
+ current = []
+ current_len = 0
+ for seg in segments:
+ line_len = len(_segment_line(seg)) + 1
+ if line_len > max_chunk_chars:
+ # A single oversized turn: flush what we have, then sentence-split
+ # the turn into its own chunks carrying its speaker and times.
+ if current:
+ chunks.append(_finalize_segment_chunk(current))
+ current, current_len = [], 0
+ for piece in chunk_transcription(seg['text'], max_chunk_length=max_chunk_chars,
+ overlap=max(50, max_chunk_chars // 20)):
+ chunks.append(_finalize_segment_chunk([{**seg, 'text': piece}]))
+ continue
+ if current and current_len + line_len > max_chunk_chars:
+ chunks.append(_finalize_segment_chunk(current))
+ # One-segment overlap so a thought spanning the boundary stays
+ # findable from either side.
+ tail = current[-1]
+ current = [tail] if len(_segment_line(tail)) + line_len <= max_chunk_chars else []
+ current_len = sum(len(_segment_line(s)) + 1 for s in current)
+ current.append(seg)
+ current_len += line_len
+ if current:
+ chunks.append(_finalize_segment_chunk(current))
+ return chunks
+
+
def chunk_transcription(transcription, max_chunk_length=500, overlap=50):
"""
Split transcription into overlapping chunks for better context retrieval.
@@ -420,7 +555,7 @@ def process_recording_chunks(recording_id):
# for the whole duration and starved concurrent writers such as the
# summarize-job enqueue (issue #355). Doing the slow work first keeps
# the delete + insert + commit window down to milliseconds.
- chunks = chunk_transcription(recording.transcription)
+ chunks = chunk_transcript_segments(recording.transcription)
if not chunks:
TranscriptChunk.query.filter_by(recording_id=recording_id).delete()
@@ -428,7 +563,7 @@ def process_recording_chunks(recording_id):
return True
# Generate embeddings (recording owner gets billed for API-mode usage)
- embeddings = generate_embeddings(chunks, user_id=recording.user_id)
+ embeddings = generate_embeddings([c['text'] for c in chunks], user_id=recording.user_id)
# Verify we got one embedding per chunk. _api_embed returns [] on
# exhausted retries, and a partial provider response could return
@@ -447,12 +582,15 @@ def process_recording_chunks(recording_id):
# only the final commit makes it permanent, so a failure here still
# rolls back to the old chunks.
TranscriptChunk.query.filter_by(recording_id=recording_id).delete()
- for i, (chunk_text, embedding) in enumerate(zip(chunks, embeddings)):
+ for i, (chunk_data, embedding) in enumerate(zip(chunks, embeddings)):
chunk = TranscriptChunk(
recording_id=recording_id,
user_id=recording.user_id,
chunk_index=i,
- content=chunk_text,
+ content=chunk_data['text'],
+ speaker_name=(chunk_data['speaker_name'] or None) and chunk_data['speaker_name'][:100],
+ start_time=chunk_data['start_time'],
+ end_time=chunk_data['end_time'],
embedding=serialize_embedding(embedding) if embedding is not None else None
)
db.session.add(chunk)
diff --git a/src/services/inquire_agent.py b/src/services/inquire_agent.py
index c91e4148..4ca23577 100644
--- a/src/services/inquire_agent.py
+++ b/src/services/inquire_agent.py
@@ -279,6 +279,7 @@ def tool_search_transcripts(ctx, args):
'meeting_date': chunk.recording.meeting_date.strftime('%Y-%m-%d') if chunk.recording.meeting_date else None,
'speaker': chunk.speaker_name,
'time': _fmt_time(chunk.start_time),
+ 'seek_seconds': int(chunk.start_time) if chunk.start_time is not None else None,
'chunk_index': chunk.chunk_index,
'snippet': chunk.content,
'similarity': round(float(similarity), 3),
@@ -565,8 +566,11 @@ Never invent content that is not in the tool results; say clearly when something
{tool_mode_prompt_block}When you give your final answer:
- Use clear markdown (headings per recording where it helps, bullets, **bold** speaker names).
-- Cite sources inline as markdown links: [Recording Title](/recordings/<recording_id>) — cite the \
-recording you drew each substantive point from, at most one citation per bullet or paragraph.
+- Cite sources inline as markdown links. When the supporting search result has a time and \
+seek_seconds, link directly to that moment: [Title @ m:ss](/recordings/<recording_id>?t=<seek_seconds>) \
+— for example [Weekly Sync @ 12:34](/recordings/73?t=754). Without a timestamp, link the recording: \
+[Title](/recordings/<recording_id>). Cite the recording each substantive point came from, at most \
+one citation per bullet or paragraph. These links open the recording and start playback at that moment.
- Order information from the most recent recordings first unless the question implies otherwise."""
diff --git a/static/js/app.modular.js b/static/js/app.modular.js
index cf55e7d4..bed4f78e 100644
--- a/static/js/app.modular.js
+++ b/static/js/app.modular.js
@@ -3877,11 +3877,39 @@ document.addEventListener('DOMContentLoaded', async () => {
// Deep link to a specific recording: /recordings/<id> (#301).
// Captured before load; selected once recordings are in.
let deepLinkRecordingId = null;
+ // Optional ?t=<seconds> seek (Inquire citations link to the
+ // cited moment): after the recording opens, seek the main
+ // player there and start playback.
+ let deepLinkSeekSeconds = null;
try {
const m = window.location.pathname.match(/^\/recordings\/(\d+)\/?$/);
if (m) deepLinkRecordingId = m[1];
+ const t = new URLSearchParams(window.location.search).get('t');
+ if (m && t !== null && /^\d+(\.\d+)?$/.test(t)) deepLinkSeekSeconds = parseFloat(t);
} catch (_) { /* best-effort */ }
+ const applyDeepLinkSeek = (seconds) => {
+ const startedAt = Date.now();
+ const attempt = () => {
+ // The detail view's player is the media element whose
+ // source is the recording's /audio/ stream.
+ const media = [...document.querySelectorAll('audio, video')]
+ .find(el => ((el.currentSrc || el.src || '').includes('/audio/')));
+ if (media && media.readyState >= 1) {
+ const target = isFinite(media.duration)
+ ? Math.min(seconds, Math.max(0, media.duration - 1))
+ : seconds;
+ media.currentTime = target;
+ // Best-effort autoplay; stays paused at the right
+ // spot if the browser blocks it.
+ media.play().catch(() => {});
+ return;
+ }
+ if (Date.now() - startedAt < 15000) setTimeout(attempt, 400);
+ };
+ setTimeout(attempt, 400);
+ };
+
// Load initial data
await Promise.all([
recordingsComposable.loadRecordings(),
@@ -3900,6 +3928,9 @@ document.addEventListener('DOMContentLoaded', async () => {
} else if (deepLinkRecordingId) {
// Honour the /recordings/<id> deep link now that data is loaded.
await recordingsComposable.selectRecordingById(deepLinkRecordingId);
+ if (deepLinkSeekSeconds !== null) {
+ applyDeepLinkSeek(deepLinkSeekSeconds);
+ }
}
// Auto-retry any uploads that previously failed and were saved to
diff --git a/tests/test_inquire_agent.py b/tests/test_inquire_agent.py
index 18a13b3c..700d4bb9 100644
--- a/tests/test_inquire_agent.py
+++ b/tests/test_inquire_agent.py
@@ -95,7 +95,8 @@ def event_types(events):
USER_CTX = {'name': 'Testy', 'title': 'analyst', 'company': 'ACME', 'output_language': None}
-SEGMENTS = [{'speaker': f'S{i % 2}', 'sentence': f'sentence number {i}', 'start_time': i * 2.0}
+SEGMENTS = [{'speaker': f'S{i % 2}', 'sentence': f'sentence number {i}',
+ 'start_time': i * 2.0, 'end_time': i * 2.0 + 1.5}
for i in range(300)]
@@ -371,6 +372,101 @@ def test_truncation_marker():
assert 'TRUNCATED' in out and 'More available.' in out
+# ---------------------------------------------------------------------------
+# Segment-aware chunking
+# ---------------------------------------------------------------------------
+
+def _seg(speaker, text, start, end):
+ return {'speaker': speaker, 'sentence': text, 'start_time': start, 'end_time': end}
+
+
+def test_chunking_quickfire_exchange_stays_together():
+ """Many short turns pack into ONE chunk with every speaker labeled inline;
+ chunk boundaries are size-based, never per-turn."""
+ from src.services.embeddings import chunk_transcript_segments
+ segs = []
+ t = 0.0
+ for i in range(20):
+ who = 'Alice' if i % 2 == 0 else 'Bob'
+ segs.append(_seg(who, f'quick reply {i}', t, t + 1.5))
+ t += 2.0
+ chunks = chunk_transcript_segments(json.dumps(segs), max_chunk_chars=1400)
+ assert len(chunks) == 1
+ c = chunks[0]
+ assert c['text'].count('Alice:') == 10 and c['text'].count('Bob:') == 10
+ assert c['start_time'] == 0.0 and c['end_time'] == pytest.approx(39.5)
+ assert c['speaker_name'] in ('Alice', 'Bob')
+
+
+def test_chunking_dominant_speaker_and_times():
+ from src.services.embeddings import chunk_transcript_segments
+ segs = [
+ _seg('Alice', 'point one', 10.0, 12.0),
+ _seg('Alice', 'point two', 12.0, 14.0),
+ _seg('Bob', 'brief interjection', 14.0, 15.0),
+ ]
+ chunks = chunk_transcript_segments(json.dumps(segs), max_chunk_chars=1400)
+ assert len(chunks) == 1
+ assert chunks[0]['speaker_name'] == 'Alice' # dominant
+ assert chunks[0]['start_time'] == 10.0 and chunks[0]['end_time'] == 15.0
+ assert 'Bob: brief interjection' in chunks[0]['text']
+
+
+def test_chunking_splits_on_size_with_segment_overlap():
+ from src.services.embeddings import chunk_transcript_segments
+ segs = [_seg('S1' if i % 2 == 0 else 'S2', f'sentence {i} ' + 'word ' * 20, i * 10.0, i * 10.0 + 8)
+ for i in range(12)]
+ chunks = chunk_transcript_segments(json.dumps(segs), max_chunk_chars=500)
+ assert len(chunks) > 1
+ # No chunk exceeds the cap; whole turns are never split across chunks
+ for c in chunks:
+ assert len(c['text']) <= 500 + 50
+ for line in c['text'].split('\n'):
+ assert line.startswith(('S1: ', 'S2: '))
+ # One-segment overlap: last line of chunk N reappears as first line of N+1
+ for a, b in zip(chunks, chunks[1:]):
+ assert a['text'].split('\n')[-1] == b['text'].split('\n')[0]
+ # Times track the segments actually in the chunk
+ assert chunks[0]['start_time'] == 0.0
+ assert chunks[-1]['end_time'] == segs[-1]['end_time']
+
+
+def test_chunking_oversized_single_turn_is_sentence_split():
+ from src.services.embeddings import chunk_transcript_segments
+ monologue = ('This is a long sentence about the quarterly results. ' * 30).strip()
+ segs = [_seg('Speaker', monologue, 5.0, 300.0)]
+ chunks = chunk_transcript_segments(json.dumps(segs), max_chunk_chars=400)
+ assert len(chunks) > 1
+ for c in chunks:
+ assert c['speaker_name'] == 'Speaker'
+ assert c['start_time'] == 5.0 and c['end_time'] == 300.0
+
+
+def test_chunking_plain_text_fallback_has_null_metadata():
+ from src.services.embeddings import chunk_transcript_segments
+ chunks = chunk_transcript_segments('just a plain text transcript. ' * 100, max_chunk_chars=500)
+ assert len(chunks) > 1
+ assert all(c['speaker_name'] is None and c['start_time'] is None for c in chunks)
+
+
+def test_process_recording_chunks_populates_metadata(data, monkeypatch):
+ import numpy as np
+ from src.services import embeddings as emb
+ with app.app_context():
+ monkeypatch.setattr(emb, 'generate_embeddings',
+ lambda texts, user_id=None: [np.zeros(8, dtype=np.float32) for _ in texts])
+ assert emb.process_recording_chunks(data['r1']) is True
+ from src.models import TranscriptChunk
+ chunks = TranscriptChunk.query.filter_by(recording_id=data['r1']).order_by(TranscriptChunk.chunk_index).all()
+ assert chunks
+ assert chunks[0].speaker_name in ('S0', 'S1')
+ assert chunks[0].start_time == 0.0
+ assert chunks[0].end_time is not None
+ assert 'S0: sentence number 0' in chunks[0].content
+ TranscriptChunk.query.filter_by(recording_id=data['r1']).delete()
+ db.session.commit()
+
+
# ---------------------------------------------------------------------------
# History and compaction
# ---------------------------------------------------------------------------