Speakr is a personal, self-hosted web application designed for transcribing audio recordings
Python
"""
Embedding generation and semantic search services.
"""
import os
import re
import json
import time
import random
from urllib.parse import urlsplit
import numpy as np
from flask import current_app
from sqlalchemy.orm import joinedload
try:
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity
LOCAL_EMBEDDINGS_AVAILABLE = True
except ImportError:
LOCAL_EMBEDDINGS_AVAILABLE = False
SentenceTransformer = None # type: ignore
cosine_similarity = None
# sklearn's cosine_similarity may be available even when sentence-transformers
# is not (e.g., the lite image with API-mode embeddings). Try to import it
# independently so semantic search can still run on API-generated vectors.
if cosine_similarity is None:
try:
from sklearn.metrics.pairwise import cosine_similarity # type: ignore
except ImportError:
cosine_similarity = None
from src.database import db
from src.models import Recording, TranscriptChunk, InternalShare, RecordingTag
from src.utils.search import LIKE_ESCAPE, contains_pattern
ENABLE_INTERNAL_SHARING = os.environ.get('ENABLE_INTERNAL_SHARING', 'false').lower() == 'true'
# Embedding model used to generate semantic-search vectors. By default Speakr
# loads sentence-transformers locally; setting EMBEDDING_BASE_URL switches to
# an OpenAI-compatible HTTP endpoint (vLLM, OpenRouter, OpenAI directly, etc.)
# and the same EMBEDDING_MODEL value is sent as the model name in requests.
# Changing the model or the endpoint after chunks are already embedded will
# produce a dimensionality / semantic-space mismatch; see the startup warning
# in src/init_db.py and the docs for guidance.
EMBEDDING_MODEL = os.environ.get('EMBEDDING_MODEL', 'all-MiniLM-L6-v2').strip() or 'all-MiniLM-L6-v2'
EMBEDDING_BASE_URL = os.environ.get('EMBEDDING_BASE_URL', '').strip()
EMBEDDING_API_KEY = os.environ.get('EMBEDDING_API_KEY', '').strip()
_dim_raw = os.environ.get('EMBEDDING_DIMENSIONS', '').strip()
try:
EMBEDDING_DIMENSIONS = int(_dim_raw) if _dim_raw else None
except (TypeError, ValueError):
EMBEDDING_DIMENSIONS = None
# When EMBEDDING_BASE_URL is set, embeddings are produced via an OpenAI-
# compatible /v1/embeddings call rather than locally.
USE_API_EMBEDDINGS = bool(EMBEDDING_BASE_URL)
# Match the official host, never a substring in a path or lookalike domain.
_USE_VOYAGE_API = urlsplit(EMBEDDING_BASE_URL).hostname == "api.voyageai.com"
# Identifier persisted to system_setting so the dimensionality compatibility
# check covers both "model changed" and "provider changed" scenarios.
EMBEDDING_IDENTIFIER = f"{EMBEDDING_BASE_URL or 'local'}::{EMBEDDING_MODEL}"
# True when at least one embedding path can produce vectors. The legacy name
# EMBEDDINGS_AVAILABLE is preserved for callers that import it elsewhere.
EMBEDDINGS_AVAILABLE = LOCAL_EMBEDDINGS_AVAILABLE or (USE_API_EMBEDDINGS and cosine_similarity is not None)
# Initialize embedding model (lazy loading)
_embedding_model = None
_embedding_api_client = None
def get_embedding_model():
"""Get or initialize the local sentence transformer model.
Returns None when running in API mode or when sentence-transformers is
not installed.
"""
global _embedding_model
if USE_API_EMBEDDINGS or not LOCAL_EMBEDDINGS_AVAILABLE:
return None
if _embedding_model is None:
try:
_embedding_model = SentenceTransformer(EMBEDDING_MODEL)
current_app.logger.info(f"Embedding model loaded successfully: {EMBEDDING_MODEL}")
except Exception as e:
current_app.logger.error(f"Failed to load embedding model {EMBEDDING_MODEL!r}: {e}")
return None
return _embedding_model
def get_embedding_api_client():
"""Get or initialize the OpenAI-compatible embeddings API client.
Returns None when API mode is disabled or the openai package is missing.
"""
global _embedding_api_client
if not USE_API_EMBEDDINGS:
return None
if _embedding_api_client is None:
try:
from openai import OpenAI
from src.services.llm import llm_timeout, LLM_MAX_RETRIES, http_client_no_proxy
_embedding_api_client = OpenAI(
api_key=EMBEDDING_API_KEY or "not-needed",
base_url=EMBEDDING_BASE_URL,
http_client=http_client_no_proxy,
timeout=llm_timeout,
max_retries=LLM_MAX_RETRIES,
)
current_app.logger.info(
f"Embedding API client initialized: base_url={EMBEDDING_BASE_URL}, model={EMBEDDING_MODEL}"
)
except Exception as e:
current_app.logger.error(f"Failed to initialize embedding API client: {e}")
return None
return _embedding_api_client
_API_EMBED_MAX_ATTEMPTS = int(os.environ.get('EMBEDDING_API_MAX_RETRIES', '3'))
# After a full retry cycle fails, skip the embedding API for this long so
# every search doesn't pay the retry latency while the provider is down.
_API_EMBED_COOLDOWN_SECONDS = int(os.environ.get('EMBEDDING_API_COOLDOWN_SECONDS', '120'))
_api_embed_down_until = 0.0
_API_EMBED_BASE_BACKOFF_SECONDS = float(os.environ.get('EMBEDDING_API_BACKOFF_SECONDS', '1.5'))
# Substrings of error messages that suggest the failure is transient and
# worth retrying. Auth and model-not-found errors do not match and fail fast.
_TRANSIENT_ERROR_HINTS = (
'timeout', 'timed out', 'connection', 'connect',
'rate limit', 'rate_limit', 'too many requests', '429',
'500', '502', '503', '504',
'temporarily unavailable', 'service unavailable',
'overloaded', 'try again',
)
def _is_transient_embedding_error(exc):
"""Decide whether an embedding-API error is worth retrying."""
msg = str(exc).lower()
return any(hint in msg for hint in _TRANSIENT_ERROR_HINTS)
# Set to True the first time the provider rejects the 'dimensions' parameter
# (models without Matryoshka support, e.g. BAAI/bge-m3, return a 400). Later
# calls then omit the parameter instead of failing the whole pipeline. Probing
# the provider beats a hardcoded list of dimension-capable model names, which
# would silently disable a setting that newer models do support.
_dimensions_rejected = False
def _is_dimensions_rejection(exc):
"""True when the error looks like the model rejecting the 'dimensions' kwarg."""
return 'dimension' in str(exc).lower()
def _api_embed(texts, user_id=None, input_type="document"):
"""Call the OpenAI-compatible embeddings endpoint and return numpy vectors.
Retries on transient errors (rate limits, timeouts, 5xx, connection
blips) with exponential backoff. Auth or model-not-found errors fail
fast since retrying will not help. On permanent failure, returns an
empty list so callers see a clear sentinel; ``process_recording_chunks``
treats a length mismatch between input and output as a failure and
rolls back, so the recording's existing chunks are preserved instead
of being silently deleted with nothing inserted in their place.
Retry parameters are tunable via ``EMBEDDING_API_MAX_RETRIES`` (default
3) and ``EMBEDDING_API_BACKOFF_SECONDS`` (default 1.5).
When ``user_id`` is provided and the response includes a ``usage``
block, record the call against the daily token-usage aggregate.
"""
global _dimensions_rejected, _api_embed_down_until
client = get_embedding_api_client()
if client is None or not texts:
return []
# Batch large inputs: providers cap the number of inputs per embeddings
# request (Gemini far lower than OpenAI), and a long recording can have
# hundreds of chunks. All-or-nothing semantics are preserved: any failed
# batch fails the whole call so callers keep their existing chunks.
batch_size = max(1, int(os.environ.get('EMBEDDING_API_BATCH_SIZE', '96')))
if len(texts) > batch_size:
out = []
for i in range(0, len(texts), batch_size):
part = _api_embed(texts[i:i + batch_size], user_id=user_id, input_type=input_type)
if len(part) != len(texts[i:i + batch_size]):
return []
out.extend(part)
return out
# Circuit breaker: after a full retry cycle fails, skip the API entirely
# for a cooldown window instead of burning ~9s of retries on every call
# (searches fall back to keyword matching immediately; indexing fails
# fast with the usual empty-list sentinel and is retried later).
if time.time() < _api_embed_down_until:
current_app.logger.debug(
"Embedding API in failure cooldown; skipping call and using fallback")
return []
if _USE_VOYAGE_API:
# Voyage accepts null/base64, not OpenAI's "float". Explicit null
# requests JSON floats and prevents the SDK's implicit base64 path.
extra_body = {'input_type': input_type}
if EMBEDDING_DIMENSIONS is not None:
extra_body['output_dimension'] = EMBEDDING_DIMENSIONS
kwargs = {'input': texts, 'model': EMBEDDING_MODEL,
'encoding_format': None, 'extra_body': extra_body}
else:
# Explicit floats also work with providers that cannot emit base64.
kwargs = {'input': texts, 'model': EMBEDDING_MODEL, 'encoding_format': 'float'}
if EMBEDDING_DIMENSIONS is not None and not _dimensions_rejected:
kwargs['dimensions'] = EMBEDDING_DIMENSIONS
last_exc = None
for attempt in range(1, _API_EMBED_MAX_ATTEMPTS + 1):
try:
response = client.embeddings.create(**kwargs)
if user_id is not None and getattr(response, 'usage', None) is not None:
try:
usage = response.usage
# Pydantic v2 stores non-schema fields under model_extra;
# fall back to attribute access for SDKs that surface them
# directly.
extras = getattr(usage, 'model_extra', None) or {}
prompt_tokens = getattr(usage, 'prompt_tokens', 0) or 0
total_tokens = getattr(usage, 'total_tokens', prompt_tokens) or prompt_tokens
cost = extras.get('cost') if 'cost' in extras else getattr(usage, 'cost', None)
from src.services.token_tracking import token_tracker
token_tracker.record_usage(
user_id=user_id,
operation_type='embedding',
prompt_tokens=int(prompt_tokens),
completion_tokens=0,
total_tokens=int(total_tokens),
model_name=EMBEDDING_MODEL,
cost=float(cost) if cost is not None else None,
)
except Exception as track_err:
current_app.logger.warning(f"Failed to record embedding usage: {track_err}")
_api_embed_down_until = 0
return [np.array(d.embedding, dtype=np.float32) for d in response.data]
except Exception as e:
last_exc = e
if 'dimensions' in kwargs and _is_dimensions_rejection(e):
_dimensions_rejected = True
kwargs.pop('dimensions')
current_app.logger.warning(
f"Embedding model '{EMBEDDING_MODEL}' rejected the 'dimensions' "
f"parameter (EMBEDDING_DIMENSIONS={EMBEDDING_DIMENSIONS}): {e}. "
"Retrying without it; the setting will be ignored for the rest "
"of this process."
)
continue
transient = _is_transient_embedding_error(e)
if not transient or attempt == _API_EMBED_MAX_ATTEMPTS:
current_app.logger.error(
f"Embedding API call failed (attempt {attempt}/{_API_EMBED_MAX_ATTEMPTS}, "
f"transient={transient}): {e}"
)
_api_embed_down_until = time.time() + _API_EMBED_COOLDOWN_SECONDS
return []
# Exponential backoff with light jitter so concurrent retries
# do not all hit the provider at the same instant.
backoff = _API_EMBED_BASE_BACKOFF_SECONDS * (2 ** (attempt - 1))
backoff *= 0.85 + 0.3 * random.random()
current_app.logger.warning(
f"Embedding API transient error (attempt {attempt}/{_API_EMBED_MAX_ATTEMPTS}), "
f"retrying in {backoff:.1f}s: {e}"
)
time.sleep(backoff)
# Loop exhausted without returning a result.
current_app.logger.error(f"Embedding API call exhausted retries: {last_exc}")
return []
# Segment-aware chunking target. ~1400 chars ≈ 300-350 tokens: large enough
# that a rapid back-and-forth exchange stays together in one chunk, small
# enough to stay a precise retrieval unit.
CHUNK_TARGET_CHARS = int(os.environ.get('CHUNK_TARGET_CHARS', '1400'))
def build_transcript_segments(transcription):
"""Parse a stored transcription into [{speaker, text, start, end}].
Returns None when the transcription is not the JSON segment format
(plain-text transcripts fall back to character windowing).
"""
if not transcription:
return None
try:
data = json.loads(transcription)
except (ValueError, TypeError):
return None
if not isinstance(data, list):
return None
segments = []
for seg in data:
if not isinstance(seg, dict):
continue
text = (seg.get('sentence') or seg.get('text') or '').strip()
if not text:
continue
segments.append({
'speaker': (seg.get('speaker') or '').strip() or None,
'text': text,
'start': seg.get('start_time'),
'end': seg.get('end_time'),
})
return segments if segments else None
def _segment_line(seg):
"""Render one segment as a transcript-style line: 'Speaker: text'."""
return f"{seg['speaker']}: {seg['text']}" if seg['speaker'] else seg['text']
def _finalize_segment_chunk(segs):
"""Build a chunk dict from a run of consecutive segments."""
text = "\n".join(_segment_line(s) for s in segs)
speakers = [s['speaker'] for s in segs if s['speaker']]
# Dominant speaker goes in the metadata column (single exact name, so the
# speaker-rename maintenance which matches by equality keeps working);
# every speaker remains visible inline in the chunk text itself.
dominant = max(set(speakers), key=speakers.count) if speakers else None
starts = [s['start'] for s in segs if s['start'] is not None]
ends = [s['end'] for s in segs if s['end'] is not None]
return {
'text': text,
'speaker_name': dominant,
'start_time': min(starts) if starts else None,
'end_time': max(ends) if ends else None,
}
def chunk_transcript_segments(transcription, max_chunk_chars=None):
"""Segment-aware chunking for diarized (JSON) transcripts.
Strategy: render each segment as a 'Speaker: text' line and pack WHOLE
segments into chunks up to ``max_chunk_chars``, never splitting a
speaker turn across chunks (except single turns longer than the limit,
which are sentence-split on their own). Chunk boundaries are size-based,
not turn-based, so a quickfire exchange of many short turns lands
together in one chunk with every speaker labeled inline. Consecutive
chunks overlap by one segment for continuity.
Returns a list of chunk dicts: {text, speaker_name (dominant),
start_time, end_time}. Plain-text transcripts fall back to the classic
character windows with null metadata.
"""
# Backwards-compatibility escape hatch: CHUNKING_STRATEGY=legacy keeps
# the pre-segment-aware behavior byte-identical (500-char windows over
# the stored string, no speaker/time metadata) for admins who do not
# want indexing behavior to change. Existing chunks are never touched
# by an upgrade either way; the strategy only applies when a recording
# is (re)indexed.
if os.environ.get('CHUNKING_STRATEGY', 'segment').strip().lower() == 'legacy':
return [{'text': t, 'speaker_name': None, 'start_time': None, 'end_time': None}
for t in chunk_transcription(transcription)]
max_chunk_chars = max_chunk_chars or CHUNK_TARGET_CHARS
segments = build_transcript_segments(transcription)
if segments is None:
return [{'text': t, 'speaker_name': None, 'start_time': None, 'end_time': None}
for t in chunk_transcription(transcription, max_chunk_length=max_chunk_chars,
overlap=max(50, max_chunk_chars // 20))]
chunks = []
current = []
current_len = 0
for seg in segments:
line_len = len(_segment_line(seg)) + 1
if line_len > max_chunk_chars:
# A single oversized turn: flush what we have, then sentence-split
# the turn into its own chunks carrying its speaker and times.
if current:
chunks.append(_finalize_segment_chunk(current))
current, current_len = [], 0
for piece in chunk_transcription(seg['text'], max_chunk_length=max_chunk_chars,
overlap=max(50, max_chunk_chars // 20)):
chunks.append(_finalize_segment_chunk([{**seg, 'text': piece}]))
continue
if current and current_len + line_len > max_chunk_chars:
chunks.append(_finalize_segment_chunk(current))
# One-segment overlap so a thought spanning the boundary stays
# findable from either side.
tail = current[-1]
current = [tail] if len(_segment_line(tail)) + line_len <= max_chunk_chars else []
current_len = sum(len(_segment_line(s)) + 1 for s in current)
current.append(seg)
current_len += line_len
if current:
chunks.append(_finalize_segment_chunk(current))
return chunks
def chunk_transcription(transcription, max_chunk_length=500, overlap=50):
"""
Split transcription into overlapping chunks for better context retrieval.
Args:
transcription (str): The full transcription text
max_chunk_length (int): Maximum characters per chunk
overlap (int): Character overlap between chunks
Returns:
list: List of text chunks
"""
if not transcription or len(transcription) <= max_chunk_length:
return [transcription] if transcription else []
chunks = []
start = 0
while start < len(transcription):
end = start + max_chunk_length
# Try to break at sentence boundaries
if end < len(transcription):
# Look for sentence endings within the last 100 characters
sentence_end = -1
for i in range(max(0, end - 100), end):
if transcription[i] in '.!?':
# Check if it's not an abbreviation
if i + 1 < len(transcription) and transcription[i + 1].isspace():
sentence_end = i + 1
if sentence_end > start:
end = sentence_end
chunk = transcription[start:end].strip()
if chunk:
chunks.append(chunk)
# Move start position with overlap
start = max(start + 1, end - overlap)
# Prevent infinite loop
if start >= len(transcription):
break
return chunks
def generate_embeddings(texts, user_id=None):
"""
Generate embeddings for a list of texts.
Routes through the OpenAI-compatible API when EMBEDDING_BASE_URL is set;
otherwise uses the locally loaded sentence-transformers model.
Args:
texts (list): List of text strings
user_id (int, optional): User to attribute the API call to for token
and cost tracking. Local-mode calls do not consume billable usage
and ignore this argument.
Returns:
list: List of embedding vectors as numpy arrays, or empty list if no
embedding path is available.
"""
if not texts:
return []
if USE_API_EMBEDDINGS:
return _api_embed(texts, user_id=user_id)
if not LOCAL_EMBEDDINGS_AVAILABLE:
current_app.logger.warning("Embeddings not available - skipping embedding generation")
return []
model = get_embedding_model()
if not model:
return []
try:
embeddings = model.encode(texts)
return [embedding.astype(np.float32) for embedding in embeddings]
except Exception as e:
current_app.logger.error(f"Error generating embeddings: {e}")
return []
def serialize_embedding(embedding):
"""Convert numpy array to binary for database storage."""
if embedding is None or not EMBEDDINGS_AVAILABLE:
return None
return embedding.tobytes()
def deserialize_embedding(binary_data):
"""Convert binary data back to numpy array."""
if binary_data is None or not EMBEDDINGS_AVAILABLE:
return None
return np.frombuffer(binary_data, dtype=np.float32)
def get_accessible_recording_ids(user_id):
"""
Get all recording IDs that a user has access to.
Includes:
- Recordings owned by the user
- Recordings shared with the user via InternalShare
- Recordings shared via group tags (if team membership exists)
Args:
user_id (int): User ID to check access for
Returns:
list: List of recording IDs the user can access
"""
accessible_ids = set()
# 1. User's own recordings
own_recordings = db.session.query(Recording.id).filter_by(user_id=user_id).all()
accessible_ids.update([r.id for r in own_recordings])
# 2. Internally shared recordings
if ENABLE_INTERNAL_SHARING:
shared_recordings = db.session.query(InternalShare.recording_id).filter_by(
shared_with_user_id=user_id
).all()
accessible_ids.update([r.recording_id for r in shared_recordings])
return list(accessible_ids)
def process_recording_chunks(recording_id):
"""
Process a recording by creating chunks and generating embeddings.
Returns True on full success, False on any failure including the case
where embedding generation returned fewer vectors than there were
chunks. Embeddings are generated before the old chunks are deleted, so
a failure leaves the recording's existing chunks intact and never holds
a database write lock across the (potentially slow) embedding API call.
"""
try:
recording = db.session.get(Recording, recording_id)
if not recording or not recording.transcription:
return False
# Create chunks and generate their embeddings BEFORE touching the
# transcript_chunk table. The embeddings call is a network round-trip
# that can take 30+ seconds against a cold local model; running it
# inside an open write transaction held SQLite's single writer lock
# for the whole duration and starved concurrent writers such as the
# summarize-job enqueue (issue #355). Doing the slow work first keeps
# the delete + insert + commit window down to milliseconds.
chunks = chunk_transcript_segments(recording.transcription)
if not chunks:
TranscriptChunk.query.filter_by(recording_id=recording_id).delete()
db.session.commit()
return True
# Generate embeddings (recording owner gets billed for API-mode usage)
embeddings = generate_embeddings([c['text'] for c in chunks], user_id=recording.user_id)
# Verify we got one embedding per chunk. _api_embed returns [] on
# exhausted retries, and a partial provider response could return
# fewer than expected. Either case is a failure; the recording's
# existing chunks are untouched so the admin retry pass (or a later
# Re-embed all) can try again.
if len(embeddings) != len(chunks):
current_app.logger.error(
f"Embedding generation returned {len(embeddings)} vectors for "
f"{len(chunks)} chunks on recording {recording_id}; keeping "
f"existing chunks."
)
return False
# Swap the chunks in one short transaction: the delete is staged and
# only the final commit makes it permanent, so a failure here still
# rolls back to the old chunks.
TranscriptChunk.query.filter_by(recording_id=recording_id).delete()
for i, (chunk_data, embedding) in enumerate(zip(chunks, embeddings)):
chunk = TranscriptChunk(
recording_id=recording_id,
user_id=recording.user_id,
chunk_index=i,
content=chunk_data['text'],
speaker_name=(chunk_data['speaker_name'] or None) and chunk_data['speaker_name'][:100],
start_time=chunk_data['start_time'],
end_time=chunk_data['end_time'],
embedding=serialize_embedding(embedding) if embedding is not None else None
)
db.session.add(chunk)
db.session.commit()
current_app.logger.info(f"Created {len(chunks)} chunks for recording {recording_id}")
return True
except Exception as e:
current_app.logger.error(f"Error processing chunks for recording {recording_id}: {e}")
db.session.rollback()
return False
def basic_text_search_chunks(user_id, query, filters=None, top_k=5):
"""
Basic text search fallback when embeddings are not available.
Uses simple text matching instead of semantic search.
Searches across user's own recordings and recordings shared with them.
"""
try:
# Get all accessible recording IDs (own + shared)
accessible_recording_ids = get_accessible_recording_ids(user_id)
if not accessible_recording_ids:
return []
# Build base query for chunks from accessible recordings with eager loading
chunks_query = TranscriptChunk.query.options(joinedload(TranscriptChunk.recording)).filter(
TranscriptChunk.recording_id.in_(accessible_recording_ids)
)
# Apply filters if provided. The tag, speaker, and date filters all need
# a join to Recording; join it at most once so combining more than one of
# them does not raise a duplicate-JOIN / ambiguous-relationship error.
if filters:
if (filters.get('tag_ids') or filters.get('speaker_names')
or filters.get('date_from') or filters.get('date_to')):
chunks_query = chunks_query.join(Recording)
if filters.get('tag_ids'):
chunks_query = chunks_query.join(
RecordingTag, Recording.id == RecordingTag.recording_id
).filter(RecordingTag.tag_id.in_(filters['tag_ids']))
if filters.get('folder_ids'):
# A subquery, so that no second join to Recording is needed (#428)
chunks_query = chunks_query.filter(TranscriptChunk.recording_id.in_(
db.session.query(Recording.id).filter(Recording.folder_id.in_(filters['folder_ids']))))
if filters.get('speaker_names'):
# Filter by the participants field on the recording.
speaker_conditions = [
Recording.participants.ilike(contains_pattern(speaker_name), escape=LIKE_ESCAPE)
for speaker_name in filters['speaker_names']
]
chunks_query = chunks_query.filter(db.or_(*speaker_conditions))
current_app.logger.info(f"Applied speaker filter for: {filters['speaker_names']}")
if filters.get('recording_ids'):
chunks_query = chunks_query.filter(
TranscriptChunk.recording_id.in_(filters['recording_ids'])
)
if filters.get('date_from') or filters.get('date_to'):
# Local calendar days of the user, as UTC bounds; date_to is
# inclusive (it used to drop the last day, #412).
from src.utils.timezones import local_date_bounds, user_timezone_by_id
_start, _end = local_date_bounds(filters.get('date_from'), filters.get('date_to'),
user_timezone_by_id(user_id))
if _start is not None:
chunks_query = chunks_query.filter(Recording.meeting_date >= _start)
if _end is not None:
chunks_query = chunks_query.filter(Recording.meeting_date < _end)
# Text search - filter stop words and rank by match count
stop_words = {'a', 'an', 'the', 'is', 'are', 'was', 'were', 'be', 'been',
'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will',
'would', 'could', 'should', 'may', 'might', 'shall', 'can',
'to', 'of', 'in', 'for', 'on', 'with', 'at', 'by', 'from',
'up', 'about', 'into', 'through', 'during', 'before', 'after',
'and', 'but', 'or', 'nor', 'not', 'so', 'yet', 'both',
'it', 'its', 'this', 'that', 'these', 'those', 'what', 'which',
'who', 'whom', 'how', 'when', 'where', 'why',
'i', 'me', 'my', 'we', 'our', 'you', 'your', 'he', 'she',
'his', 'her', 'they', 'them', 'their'}
query_words = [w for w in query.lower().split() if w not in stop_words and len(w) > 1]
if not query_words:
# If all words were stop words, fall back to using original query words
query_words = [w for w in query.lower().split() if len(w) > 1]
if query_words:
from sqlalchemy import or_, case
# Filter: match ANY keyword (OR) to get candidates
text_conditions = []
for word in query_words:
text_conditions.append(TranscriptChunk.content.ilike(contains_pattern(word), escape=LIKE_ESCAPE))
chunks_query = chunks_query.filter(or_(*text_conditions))
# Rank IN SQL before limiting. The previous code applied
# LIMIT before any ranking, so with a common word in the query
# ("rate", "team") the candidate set filled up with arbitrary
# low-id chunks and the actually-relevant chunks never entered
# the ranking at all. Score = number of matched words, with the
# full phrase counting double so exact-phrase hits always
# outrank scattered single-word matches.
score_expr = sum(
case((TranscriptChunk.content.ilike(contains_pattern(word), escape=LIKE_ESCAPE), 1), else_=0)
for word in query_words
)
phrase = query.lower().strip()
if len(query_words) > 1 and len(phrase) >= 5:
score_expr = score_expr + case(
(TranscriptChunk.content.ilike(contains_pattern(phrase), escape=LIKE_ESCAPE), len(query_words)),
else_=0,
)
chunks = (chunks_query
.order_by(score_expr.desc(), TranscriptChunk.id.desc())
.limit(top_k * 5)
.all())
# Refine in Python over the now well-chosen candidates: exact
# word-boundary matches beat substring matches ("rate" inside
# "separate"), and phrase presence keeps its bonus.
scored_chunks = []
for chunk in chunks:
content_lower = chunk.content.lower()
match_count = 0.0
for word in query_words:
if re.search(r'\b' + re.escape(word) + r'\b', content_lower):
match_count += 1.0
elif word in content_lower:
match_count += 0.5
if len(query_words) > 1 and phrase in content_lower:
match_count += len(query_words)
# Full word-boundary matches on every word score 1.0; the
# phrase bonus only breaks ties above, capped back to 1.0.
score = min(match_count / len(query_words), 1.0)
scored_chunks.append((chunk, score))
# Sort by score descending, take top_k
scored_chunks.sort(key=lambda x: x[1], reverse=True)
return scored_chunks[:top_k]
# No usable query words
return []
except Exception as e:
current_app.logger.error(f"Error in basic text search: {e}")
return []
def semantic_search_chunks(user_id, query, filters=None, top_k=5):
"""
Perform semantic search on transcript chunks with filtering.
Searches across user's own recordings and recordings shared with them.
Args:
user_id (int): User ID for permission filtering
query (str): Search query
filters (dict): Optional filters for tags, speakers, dates, recording_ids
top_k (int): Number of top chunks to return
Returns:
list: List of relevant chunks with similarity scores
"""
try:
# If embeddings are not available, fall back to basic text search
if not EMBEDDINGS_AVAILABLE:
current_app.logger.info("Embeddings not available - using basic text search as fallback")
return basic_text_search_chunks(user_id, query, filters, top_k)
if cosine_similarity is None:
current_app.logger.info("scikit-learn not installed - using basic text search as fallback")
return basic_text_search_chunks(user_id, query, filters, top_k)
# Generate embedding for the query (via API or local model). Attribute
# the API call to the searching user so embedding cost tracking shows
# who issued the query.
if USE_API_EMBEDDINGS:
api_vectors = _api_embed([query], user_id=user_id, input_type="query")
if not api_vectors:
return basic_text_search_chunks(user_id, query, filters, top_k)
query_embedding = api_vectors[0]
else:
model = get_embedding_model()
if not model:
return basic_text_search_chunks(user_id, query, filters, top_k)
query_embedding = model.encode([query])[0]
# Get all accessible recording IDs (own + shared)
accessible_recording_ids = get_accessible_recording_ids(user_id)
if not accessible_recording_ids:
return []
# Build base query for chunks from accessible recordings with eager loading
chunks_query = TranscriptChunk.query.options(joinedload(TranscriptChunk.recording)).filter(
TranscriptChunk.recording_id.in_(accessible_recording_ids)
)
# Apply filters if provided. The tag, speaker, and date filters all need
# a join to Recording; join it at most once so combining more than one of
# them does not raise a duplicate-JOIN / ambiguous-relationship error.
if filters:
if (filters.get('tag_ids') or filters.get('speaker_names')
or filters.get('date_from') or filters.get('date_to')):
chunks_query = chunks_query.join(Recording)
if filters.get('tag_ids'):
chunks_query = chunks_query.join(
RecordingTag, Recording.id == RecordingTag.recording_id
).filter(RecordingTag.tag_id.in_(filters['tag_ids']))
if filters.get('folder_ids'):
# A subquery, so that no second join to Recording is needed (#428)
chunks_query = chunks_query.filter(TranscriptChunk.recording_id.in_(
db.session.query(Recording.id).filter(Recording.folder_id.in_(filters['folder_ids']))))
if filters.get('speaker_names'):
# Filter by the participants field on the recording.
speaker_conditions = [
Recording.participants.ilike(contains_pattern(speaker_name), escape=LIKE_ESCAPE)
for speaker_name in filters['speaker_names']
]
chunks_query = chunks_query.filter(db.or_(*speaker_conditions))
current_app.logger.info(f"Applied speaker filter for: {filters['speaker_names']}")
if filters.get('recording_ids'):
chunks_query = chunks_query.filter(
TranscriptChunk.recording_id.in_(filters['recording_ids'])
)
if filters.get('date_from') or filters.get('date_to'):
# Local calendar days of the user, as UTC bounds; date_to is
# inclusive (it used to drop the last day, #412).
from src.utils.timezones import local_date_bounds, user_timezone_by_id
_start, _end = local_date_bounds(filters.get('date_from'), filters.get('date_to'),
user_timezone_by_id(user_id))
if _start is not None:
chunks_query = chunks_query.filter(Recording.meeting_date >= _start)
if _end is not None:
chunks_query = chunks_query.filter(Recording.meeting_date < _end)
# Get chunks that have embeddings
chunks = chunks_query.filter(TranscriptChunk.embedding.isnot(None)).all()
if not chunks:
return []
# Calculate similarities. Previously this iterated chunk-by-chunk and
# called sklearn's cosine_similarity on a 1xN vs 1xN pair for every
# chunk, which on a library of ~17k chunks took 15-20 seconds per
# query because of Python-call overhead per chunk. Stacking all
# chunk vectors into one matrix and doing a single vectorised dot
# product brings that down by two to three orders of magnitude.
expected_dim = int(query_embedding.shape[0])
kept_chunks = []
kept_vectors = []
skipped_dim_mismatch = 0
for chunk in chunks:
if chunk.embedding is None:
continue
try:
v = deserialize_embedding(chunk.embedding)
except Exception as e:
current_app.logger.warning(f"Error deserialising chunk {chunk.id}: {e}")
continue
if v is None:
continue
if v.shape[0] != expected_dim:
# Stale vector from a previous embedding configuration.
# Skip silently in batch rather than warning per chunk so a
# large library cannot flood the log on every search.
skipped_dim_mismatch += 1
continue
kept_chunks.append(chunk)
kept_vectors.append(v)
if skipped_dim_mismatch:
current_app.logger.warning(
f"Skipped {skipped_dim_mismatch} chunks with mismatched "
f"embedding dimensions (expected {expected_dim}). Run "
f"Re-embed all to refresh them."
)
if not kept_vectors:
return []
# One sklearn call instead of len(chunks). Returns shape (1, N).
embeddings_matrix = np.vstack(kept_vectors)
similarities = cosine_similarity(
query_embedding.reshape(1, -1),
embeddings_matrix,
)[0]
# Top-k via argpartition is faster than a full sort for large N.
if top_k >= len(kept_chunks):
order = np.argsort(-similarities)
else:
top_idx = np.argpartition(-similarities, top_k)[:top_k]
order = top_idx[np.argsort(-similarities[top_idx])]
return [(kept_chunks[i], float(similarities[i])) for i in order]
except Exception as e:
current_app.logger.error(f"Error in semantic search: {e}")
return []
# --- Helper Functions for Document Processing ---
def embeddings_are_local():
"""True when embeddings are computed on machines the administrator runs:
the built-in local model, or an endpoint on a loopback or private address
or a .local/.lan/.internal host. Used to show "local" in place of $0."""
if not USE_API_EMBEDDINGS:
return True
import ipaddress
from urllib.parse import urlparse
host = (urlparse(EMBEDDING_BASE_URL).hostname or '').lower()
if host in ('localhost',) or host.endswith(('.local', '.lan', '.internal', '.localhost')):
return True
try:
ip = ipaddress.ip_address(host)
return ip.is_private or ip.is_loopback or ip.is_link_local
except ValueError:
return False