Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python
31.6 KB
Raw Blame
"""
LLM API integration services (OpenAI/OpenRouter).
"""

import os
import re
import json
import time
import logging
import httpx
from openai import OpenAI, APITimeoutError, BadRequestError

# Use standard logging instead of current_app.logger for context independence
logger = logging.getLogger(__name__)


class TokenBudgetExceeded(Exception):
    """Raised when user exceeds their token budget."""
    def __init__(self, message, usage_percentage=100):
        self.message = message
        self.usage_percentage = usage_percentage
        super().__init__(message)

from src.utils import safe_json_loads, extract_json_object


def extract_cache_tokens(usage):
    """Pull prompt-cache counts out of a provider's usage object.

    Returns (cached_tokens, cache_write_tokens) as ints, defaulting to 0.
    Providers expose these differently:
      - OpenAI / OpenRouter: usage.prompt_tokens_details.cached_tokens (reads),
        and some surface a cache_write/cache_creation field alongside it.
      - Anthropic: usage.cache_read_input_tokens / usage.cache_creation_input_tokens.
    We probe all known shapes defensively because usage may be a pydantic
    object or a plain dict, and missing fields must never break tracking.
    """
    if not usage:
        return 0, 0

    def _get(obj, name):
        if obj is None:
            return None
        if isinstance(obj, dict):
            return obj.get(name)
        return getattr(obj, name, None)

    def _to_int(val):
        try:
            return int(val) if val else 0
        except (TypeError, ValueError):
            return 0

    details = _get(usage, 'prompt_tokens_details')

    cached = (
        _get(details, 'cached_tokens')
        or _get(usage, 'cache_read_input_tokens')
        or 0
    )
    cache_write = (
        _get(usage, 'cache_creation_input_tokens')
        or _get(details, 'cache_write_tokens')
        or _get(usage, 'cache_write_tokens')
        or 0
    )
    return _to_int(cached), _to_int(cache_write)

# Configuration - use TEXT_MODEL_* variables for LLM
TEXT_MODEL_API_KEY = os.environ.get("TEXT_MODEL_API_KEY")
TEXT_MODEL_BASE_URL = os.environ.get("TEXT_MODEL_BASE_URL", "https://openrouter.ai/api/v1")
if TEXT_MODEL_BASE_URL:
    TEXT_MODEL_BASE_URL = TEXT_MODEL_BASE_URL.split('#')[0].strip()
TEXT_MODEL_NAME = os.environ.get("TEXT_MODEL_NAME", "openai/gpt-3.5-turbo")

# Chat model configuration (optional - falls back to TEXT_MODEL_* if not set)
CHAT_MODEL_API_KEY = os.environ.get("CHAT_MODEL_API_KEY")
CHAT_MODEL_BASE_URL = os.environ.get("CHAT_MODEL_BASE_URL")
if CHAT_MODEL_BASE_URL:
    CHAT_MODEL_BASE_URL = CHAT_MODEL_BASE_URL.split('#')[0].strip()
CHAT_MODEL_NAME = os.environ.get("CHAT_MODEL_NAME")

# Chat-specific GPT-5 settings (optional - falls back to main GPT5_* settings)
CHAT_GPT5_REASONING_EFFORT = os.environ.get("CHAT_GPT5_REASONING_EFFORT")
CHAT_GPT5_VERBOSITY = os.environ.get("CHAT_GPT5_VERBOSITY")

# Provider-specific request parameters, as a JSON object merged into the body
# of every request to the model (OpenAI SDK extra_body). Reasoning controls
# differ by provider and model (reasoning_effort, OpenRouter's reasoning
# object, chat_template_kwargs on vLLM), so they are set here, not mapped.
def _parse_extra_body(env_var):
    """Return (parameters, error) for a JSON object in env_var."""
    raw = os.environ.get(env_var, "").strip()
    if not raw:
        return {}, None
    try:
        value = json.loads(raw)
    except ValueError as e:
        return {}, f"{env_var} is not valid JSON ({e})"
    if not isinstance(value, dict):
        return {}, f"{env_var} must be a JSON object, such as {{\"reasoning_effort\": \"low\"}}"
    return value, None


TEXT_MODEL_EXTRA_BODY, TEXT_MODEL_EXTRA_BODY_ERROR = _parse_extra_body("TEXT_MODEL_EXTRA_BODY")
CHAT_MODEL_EXTRA_BODY, CHAT_MODEL_EXTRA_BODY_ERROR = _parse_extra_body("CHAT_MODEL_EXTRA_BODY")
for _error in (TEXT_MODEL_EXTRA_BODY_ERROR, CHAT_MODEL_EXTRA_BODY_ERROR):
    if _error:
        logger.error(f"{_error}; requests to that model will fail until it is corrected.")

# Streaming options - disable for LLM servers that don't support OpenAI's stream_options
ENABLE_STREAM_OPTIONS = os.environ.get("ENABLE_STREAM_OPTIONS", "true").lower() == "true"

# LLM timeout and retry configuration
# Read timeout controls how long to wait for model inference (the bottleneck for local models)
# Connect/write stay short so misconfigured URLs fail fast instead of hanging
LLM_REQUEST_TIMEOUT = int(os.environ.get("LLM_REQUEST_TIMEOUT", "600"))
LLM_MAX_RETRIES = int(os.environ.get("LLM_MAX_RETRIES", "2"))
LLM_CONNECT_TIMEOUT = float(os.environ.get("LLM_CONNECT_TIMEOUT", "30"))
LLM_WRITE_TIMEOUT = float(os.environ.get("LLM_WRITE_TIMEOUT", "30"))
llm_timeout = httpx.Timeout(
    connect=LLM_CONNECT_TIMEOUT,
    read=float(LLM_REQUEST_TIMEOUT),
    write=LLM_WRITE_TIMEOUT,
    pool=30.0,
)


def _gpt5_mode_forced(env_var="GPT5_FORCE_MODE"):
    """Whether GPT-5 parameter handling is forced on via environment variable.

    The autodetection requires api.openai.com in the base URL, which misses
    GPT-5 models served through Azure AI Foundry or other gateways (#367).
    """
    return str(os.environ.get(env_var, "")).strip().lower() in ("true", "1", "yes")


def get_chat_config():
    """
    Get chat model configuration, falling back to TEXT_MODEL if not set.

    Returns a dict with api_key, base_url, model_name, and GPT-5 settings.
    A dedicated chat model needs CHAT_MODEL_NAME and either its own key or
    its own base URL; a local server often needs no key at all (#403).
    """
    if CHAT_MODEL_NAME and (CHAT_MODEL_API_KEY or CHAT_MODEL_BASE_URL):
        return {
            # The text model's key goes only to the text model's server.
            'api_key': CHAT_MODEL_API_KEY or (TEXT_MODEL_API_KEY if (CHAT_MODEL_BASE_URL or TEXT_MODEL_BASE_URL) == TEXT_MODEL_BASE_URL else None),
            'base_url': CHAT_MODEL_BASE_URL or TEXT_MODEL_BASE_URL,
            'model_name': CHAT_MODEL_NAME,
            'gpt5_reasoning_effort': CHAT_GPT5_REASONING_EFFORT or os.environ.get("GPT5_REASONING_EFFORT", "medium"),
            'gpt5_verbosity': CHAT_GPT5_VERBOSITY or os.environ.get("GPT5_VERBOSITY", "medium"),
            # A separate chat model does not inherit GPT5_FORCE_MODE: it may
            # be a different (non-GPT-5) model than the main text model.
            'gpt5_force': _gpt5_mode_forced("CHAT_GPT5_FORCE_MODE"),
            # Like GPT5_FORCE_MODE, the text model's parameters are not
            # inherited: they may not suit a different model or server.
            'extra_body': CHAT_MODEL_EXTRA_BODY,
            'extra_body_variable': 'CHAT_MODEL_EXTRA_BODY',
            'extra_body_error': CHAT_MODEL_EXTRA_BODY_ERROR,
        }
    return {
        'api_key': TEXT_MODEL_API_KEY,
        'base_url': TEXT_MODEL_BASE_URL,
        'model_name': TEXT_MODEL_NAME,
        'gpt5_reasoning_effort': os.environ.get("GPT5_REASONING_EFFORT", "medium"),
        'gpt5_verbosity': os.environ.get("GPT5_VERBOSITY", "medium"),
        'gpt5_force': _gpt5_mode_forced(),
        'extra_body': TEXT_MODEL_EXTRA_BODY,
        'extra_body_variable': 'TEXT_MODEL_EXTRA_BODY',
        'extra_body_error': TEXT_MODEL_EXTRA_BODY_ERROR,
    }


# Set up HTTP client with custom headers for OpenRouter app identification
app_headers = {
    "HTTP-Referer": "https://github.com/murtaza-nasir/speakr",
    "X-Title": "Speakr - AI Audio Transcription",
    "User-Agent": "Speakr/1.0 (https://github.com/murtaza-nasir/speakr)"
}

http_client_no_proxy = httpx.Client(
    verify=True,
    headers=app_headers
)

# A language model is configured when it has a key or its server address was
# set explicitly; a local server often needs no key (#403). Without either,
# no client is created and nothing is sent: the default address is a hosted
# service, and a transcript must never go there unasked.
TEXT_MODEL_CONFIGURED = bool(TEXT_MODEL_API_KEY) or bool(os.environ.get("TEXT_MODEL_BASE_URL", "").split('#')[0].strip())

try:
    if not TEXT_MODEL_CONFIGURED:
        raise RuntimeError("no text model configured (set TEXT_MODEL_BASE_URL, and TEXT_MODEL_API_KEY for a hosted service)")
    api_key = TEXT_MODEL_API_KEY or "not-needed"
    client = OpenAI(
        api_key=api_key,
        base_url=TEXT_MODEL_BASE_URL,
        http_client=http_client_no_proxy,
        timeout=llm_timeout,
        max_retries=LLM_MAX_RETRIES,
    )
    # Always log the resolved timeout/retry values at startup so users can confirm
    # their LLM_REQUEST_TIMEOUT / LLM_MAX_RETRIES env vars actually took effect.
    logger.info(
        f"LLM client configured: read_timeout={LLM_REQUEST_TIMEOUT}s, "
        f"connect_timeout={LLM_CONNECT_TIMEOUT}s, write_timeout={LLM_WRITE_TIMEOUT}s, "
        f"max_retries={LLM_MAX_RETRIES}"
    )
except Exception as client_init_e:
    client = None

def _build_chat_client():
    """The chat client: the main client, or a dedicated one when the chat
    model has its own key or server. The key is optional, since a local
    server often runs without one (#403)."""
    chat_config = get_chat_config()
    if (chat_config['api_key'], chat_config['base_url']) == (TEXT_MODEL_API_KEY, TEXT_MODEL_BASE_URL):
        return client
    if not (CHAT_MODEL_API_KEY or CHAT_MODEL_BASE_URL or TEXT_MODEL_CONFIGURED):
        return None
    dedicated = OpenAI(
        api_key=chat_config['api_key'] or "not-needed",
        base_url=chat_config['base_url'],
        http_client=http_client_no_proxy,
        timeout=llm_timeout,
        max_retries=LLM_MAX_RETRIES,
    )
    logger.info(f"Separate chat client initialized: {chat_config['base_url']} / {chat_config['model_name']}")
    return dedicated


# Create chat client (may be same as main client if no separate config)
try:
    chat_client = _build_chat_client()
except Exception as chat_client_init_e:
    logger.warning(f"Failed to initialize chat client, falling back to main client: {chat_client_init_e}")
    chat_client = client


def _add_extra_body(completion_args, extra_body, error):
    """Merge the configured request parameters into completion_args.

    They are applied last, so they take precedence over the parameters set
    above (temperature, max_tokens, the GPT-5 settings)."""
    if error:
        raise ValueError(error)
    if extra_body:
        completion_args["extra_body"] = dict(extra_body)


def _log_rejected_parameters(error, variable, extra_body):
    """Name the request parameters setting when the provider rejects a request."""
    if extra_body and isinstance(error, BadRequestError):
        logger.error(
            f"The model server rejected the request. {variable} adds "
            f"{', '.join(sorted(extra_body))}; check that the server accepts these parameters."
        )


def is_gpt5_model(model_name):
    """
    Check if the model is a GPT-5 series model that requires special API parameters.

    Args:
        model_name: The model name string

    Returns:
        Boolean indicating if this is a GPT-5 model
    """
    if not model_name:
        return False
    model_lower = model_name.lower()
    return model_lower.startswith('gpt-5') or model_lower in ['gpt-5', 'gpt-5-mini', 'gpt-5-nano', 'gpt-5-chat-latest']



def is_using_openai_api():
    """
    Check if we're using the official OpenAI API (not OpenRouter or other providers).

    Returns:
        Boolean indicating if this is the OpenAI API
    """
    return TEXT_MODEL_BASE_URL and 'api.openai.com' in TEXT_MODEL_BASE_URL



class LLMNotConfiguredError(ValueError):
    """No text model is configured; the API answers with text_model_not_configured."""


def call_llm_completion(messages, temperature=0.7, response_format=None, stream=False, max_tokens=None,
                        user_id=None, operation_type=None):
    """
    Centralized function for LLM API calls with proper error handling and logging.

    Args:
        messages: List of message dicts with 'role' and 'content'
        temperature: Sampling temperature (0-1) - ignored for GPT-5 models
        response_format: Optional response format dict (e.g., {"type": "json_object"})
        stream: Whether to stream the response
        max_tokens: Optional maximum tokens to generate
        user_id: Optional user ID for token tracking and budget enforcement
        operation_type: Optional operation type for token tracking (e.g., 'summarization', 'chat')

    Returns:
        OpenAI completion object or generator (if streaming)
    """
    if not client:
        raise LLMNotConfiguredError("No text model is configured. Set TEXT_MODEL_BASE_URL, and TEXT_MODEL_API_KEY for a hosted service.")

    # Check budget before making the call
    if user_id and operation_type:
        try:
            from src.services.token_tracking import token_tracker
            can_proceed, usage_pct, msg = token_tracker.check_budget(user_id)
            if not can_proceed:
                raise TokenBudgetExceeded(msg, usage_pct)
            if usage_pct >= 80:
                logger.warning(f"User {user_id} at {usage_pct:.1f}% of token budget")
        except TokenBudgetExceeded:
            raise
        except Exception as e:
            # Log but don't block on budget check errors
            logger.warning(f"Budget check failed for user {user_id}: {e}")

    try:
        # Check if we're using GPT-5 with OpenAI API (or forced via env for
        # Azure AI Foundry and other gateways the URL check can't detect)
        using_gpt5 = _gpt5_mode_forced() or (is_gpt5_model(TEXT_MODEL_NAME) and is_using_openai_api())

        completion_args = {
            "model": TEXT_MODEL_NAME,
            "messages": messages,
            "stream": stream
        }

        # Add stream_options to get usage in final chunk for streaming
        # Some LLM servers don't support this OpenAI-specific option
        if stream and ENABLE_STREAM_OPTIONS:
            completion_args["stream_options"] = {"include_usage": True}

        if using_gpt5:
            # GPT-5 models don't support temperature, top_p, or logprobs
            # They use reasoning_effort and verbosity instead
            logger.debug(f"Using GPT-5 model: {TEXT_MODEL_NAME} - applying GPT-5 specific parameters")

            # Get GPT-5 specific parameters from environment variables
            reasoning_effort = os.environ.get("GPT5_REASONING_EFFORT", "medium")  # minimal, low, medium, high
            verbosity = os.environ.get("GPT5_VERBOSITY", "medium")  # low, medium, high

            # Add GPT-5 specific parameters
            completion_args["reasoning_effort"] = reasoning_effort
            completion_args["verbosity"] = verbosity

            # Use max_completion_tokens instead of max_tokens for GPT-5
            if max_tokens:
                completion_args["max_completion_tokens"] = max_tokens
        else:
            # Non-GPT-5 models use standard parameters
            completion_args["temperature"] = temperature

            if max_tokens:
                completion_args["max_tokens"] = max_tokens

        if response_format:
            completion_args["response_format"] = response_format

        _add_extra_body(completion_args, TEXT_MODEL_EXTRA_BODY, TEXT_MODEL_EXTRA_BODY_ERROR)

        # Make the resolved max_tokens visible in logs so users can confirm
        # their SUMMARY_MAX_TOKENS / CHAT_MAX_TOKENS / etc. settings actually
        # took effect for a given operation. The "key" used here is
        # max_completion_tokens for GPT-5 and max_tokens otherwise.
        budget_key = 'max_completion_tokens' if using_gpt5 else 'max_tokens'
        logger.info(
            f"LLM call: operation={operation_type or 'unspecified'}, model={TEXT_MODEL_NAME}, "
            f"{budget_key}={completion_args.get(budget_key, 'provider default')}"
        )

        request_started_at = time.monotonic()
        response = client.chat.completions.create(**completion_args)

        # Track usage for non-streaming calls
        if user_id and operation_type and not stream and response.usage:
            try:
                from src.services.token_tracking import token_tracker
                cached_tokens, cache_write_tokens = extract_cache_tokens(response.usage)
                token_tracker.record_usage(
                    user_id=user_id,
                    operation_type=operation_type,
                    prompt_tokens=response.usage.prompt_tokens,
                    completion_tokens=response.usage.completion_tokens,
                    total_tokens=response.usage.total_tokens,
                    model_name=TEXT_MODEL_NAME,
                    cost=getattr(response.usage, 'cost', None),
                    cached_tokens=cached_tokens,
                    cache_write_tokens=cache_write_tokens
                )
            except Exception as e:
                logger.warning(f"Failed to record token usage: {e}")

        # Debug log for empty responses
        if not stream and response.choices:
            content = response.choices[0].message.content
            if not content:
                logger.warning(f"LLM returned empty content. Model: {TEXT_MODEL_NAME}, finish_reason: {response.choices[0].finish_reason}")
                # Log more details if available
                if hasattr(response.choices[0].message, 'refusal'):
                    logger.warning(f"Refusal: {response.choices[0].message.refusal}")
                if hasattr(response.choices[0].message, 'tool_calls') and response.choices[0].message.tool_calls:
                    logger.warning(f"Tool calls present: {response.choices[0].message.tool_calls}")

        return response

    except TokenBudgetExceeded:
        raise
    except APITimeoutError as e:
        elapsed = time.monotonic() - request_started_at if 'request_started_at' in locals() else None
        elapsed_str = f"{elapsed:.1f}s" if elapsed is not None else "unknown"
        logger.error(
            f"LLM request timed out after {elapsed_str} (configured read_timeout={LLM_REQUEST_TIMEOUT}s, "
            f"max_retries={LLM_MAX_RETRIES}, model={TEXT_MODEL_NAME}). "
            f"If the elapsed time is much shorter than the configured timeout, an upstream proxy or the "
            f"provider is closing the connection early. For reasoning models that take longer to think, "
            f"increase LLM_REQUEST_TIMEOUT."
        )
        raise
    except Exception as e:
        logger.error(f"LLM API call failed: {e}")
        _log_rejected_parameters(e, 'TEXT_MODEL_EXTRA_BODY', TEXT_MODEL_EXTRA_BODY)
        raise


def call_chat_completion(messages, temperature=0.7, response_format=None, stream=False, max_tokens=None,
                         user_id=None, operation_type=None, tools=None, tool_choice=None):
    """
    Chat-specific LLM completion function. Uses dedicated chat model if configured,
    otherwise falls back to standard TEXT_MODEL configuration.

    Args:
        messages: List of message dicts with 'role' and 'content'
        temperature: Sampling temperature (0-1) - ignored for GPT-5 models
        response_format: Optional response format dict (e.g., {"type": "json_object"})
        stream: Whether to stream the response
        max_tokens: Optional maximum tokens to generate
        user_id: Optional user ID for token tracking and budget enforcement
        operation_type: Optional operation type for token tracking (e.g., 'chat')
        tools: Optional list of tool schemas (OpenAI function-calling format).
            Only sent when provided, so non-agent call sites are unaffected.
        tool_choice: Optional tool_choice value, passed through when tools are set.

    Returns:
        OpenAI completion object or generator (if streaming)
    """
    effective_client = chat_client if chat_client else client
    chat_config = get_chat_config()

    if not effective_client:
        raise ValueError("Chat LLM client not initialized")

    # Check budget before making the call
    if user_id and operation_type:
        try:
            from src.services.token_tracking import token_tracker
            can_proceed, usage_pct, msg = token_tracker.check_budget(user_id)
            if not can_proceed:
                raise TokenBudgetExceeded(msg, usage_pct)
            if usage_pct >= 80:
                logger.warning(f"User {user_id} at {usage_pct:.1f}% of token budget")
        except TokenBudgetExceeded:
            raise
        except Exception as e:
            # Log but don't block on budget check errors
            logger.warning(f"Budget check failed for user {user_id}: {e}")

    try:
        model_name = chat_config['model_name']
        base_url = chat_config['base_url'] or ''

        # Check if we're using GPT-5 with OpenAI API (or forced via env for
        # Azure AI Foundry and other gateways the URL check can't detect)
        using_gpt5 = chat_config['gpt5_force'] or (is_gpt5_model(model_name) and 'api.openai.com' in base_url)

        completion_args = {
            "model": model_name,
            "messages": messages,
            "stream": stream
        }

        # Add stream_options to get usage in final chunk for streaming
        # Some LLM servers don't support this OpenAI-specific option
        if stream and ENABLE_STREAM_OPTIONS:
            completion_args["stream_options"] = {"include_usage": True}

        if using_gpt5:
            logger.debug(f"Using GPT-5 chat model: {model_name}")
            # Use chat-specific GPT-5 settings from config
            completion_args["reasoning_effort"] = chat_config['gpt5_reasoning_effort']
            completion_args["verbosity"] = chat_config['gpt5_verbosity']

            if max_tokens:
                completion_args["max_completion_tokens"] = max_tokens
        else:
            completion_args["temperature"] = temperature
            if max_tokens:
                completion_args["max_tokens"] = max_tokens

        if response_format:
            completion_args["response_format"] = response_format

        if tools:
            completion_args["tools"] = tools
            if tool_choice:
                completion_args["tool_choice"] = tool_choice

        _add_extra_body(completion_args, chat_config['extra_body'], chat_config['extra_body_error'])

        # Visibility: surface the resolved budget per call so admins can
        # confirm CHAT_MAX_TOKENS or per-call overrides took effect.
        budget_key = 'max_completion_tokens' if using_gpt5 else 'max_tokens'
        logger.info(
            f"Chat LLM call: operation={operation_type or 'unspecified'}, model={model_name}, "
            f"{budget_key}={completion_args.get(budget_key, 'provider default')}"
        )

        request_started_at = time.monotonic()
        response = effective_client.chat.completions.create(**completion_args)

        # Track usage for non-streaming calls
        if user_id and operation_type and not stream and response.usage:
            try:
                from src.services.token_tracking import token_tracker
                cached_tokens, cache_write_tokens = extract_cache_tokens(response.usage)
                token_tracker.record_usage(
                    user_id=user_id,
                    operation_type=operation_type,
                    prompt_tokens=response.usage.prompt_tokens,
                    completion_tokens=response.usage.completion_tokens,
                    total_tokens=response.usage.total_tokens,
                    model_name=model_name,
                    cost=getattr(response.usage, 'cost', None),
                    cached_tokens=cached_tokens,
                    cache_write_tokens=cache_write_tokens
                )
            except Exception as e:
                logger.warning(f"Failed to record token usage: {e}")

        # Debug log for empty responses
        if not stream and response.choices:
            content = response.choices[0].message.content
            if not content:
                logger.warning(f"Chat LLM returned empty content. Model: {model_name}, finish_reason: {response.choices[0].finish_reason}")

        return response

    except TokenBudgetExceeded:
        raise
    except APITimeoutError as e:
        elapsed = time.monotonic() - request_started_at if 'request_started_at' in locals() else None
        elapsed_str = f"{elapsed:.1f}s" if elapsed is not None else "unknown"
        logger.error(
            f"Chat LLM request timed out after {elapsed_str} (configured read_timeout={LLM_REQUEST_TIMEOUT}s, "
            f"max_retries={LLM_MAX_RETRIES}, model={model_name}). "
            f"If the elapsed time is much shorter than the configured timeout, an upstream proxy or the "
            f"provider is closing the connection early. For reasoning models that take longer to think, "
            f"increase LLM_REQUEST_TIMEOUT."
        )
        raise
    except Exception as e:
        logger.error(f"Chat LLM API call failed: {e}")
        _log_rejected_parameters(e, chat_config['extra_body_variable'], chat_config['extra_body'])
        raise


def format_api_error_message(error_str):
    """
    Formats API error messages to be more user-friendly.
    Specifically handles token limit errors with helpful suggestions.
    """
    error_lower = error_str.lower()
    
    # Check for token limit errors
    if 'maximum context length' in error_lower and 'tokens' in error_lower:
        return "[Summary generation failed: The transcription is too long for AI processing. Request your admin to try using a different LLM with a larger context size, or set a limit for the transcript_length_limit in the system settings.]"
    
    # Check for other common API errors
    if 'rate limit' in error_lower:
        return "[Summary generation failed: API rate limit exceeded. Please try again in a few minutes.]"
    
    if 'insufficient funds' in error_lower or 'quota exceeded' in error_lower:
        return "[Summary generation failed: API quota exceeded. Please contact support.]"
    
    if 'timeout' in error_lower:
        return "[Summary generation failed: Request timed out. Please try again.]"
    
    # For other errors, show a generic message
    return f"[Summary generation failed: {error_str}]"


def process_streaming_with_thinking(stream, user_id=None, operation_type=None, model_name=None, app=None):
    """
    Generator that processes a streaming response and separates thinking content.
    Yields SSE-formatted data with 'delta' for regular content and 'thinking' for thinking content.

    Args:
        stream: The streaming response from the LLM API
        user_id: Optional user ID for token tracking
        operation_type: Optional operation type for token tracking
        model_name: Optional model name for token tracking
        app: Optional Flask app instance for database context in generators
    """
    content_buffer = ""
    in_thinking = False
    thinking_buffer = ""
    finish_reason = None

    for chunk in stream:
        # Capture the provider's final finish_reason so truncation
        # (finish_reason == 'length') survives to the terminal SSE event
        # instead of being silently dropped (issue #349).
        if chunk.choices:
            chunk_finish = getattr(chunk.choices[0], 'finish_reason', None)
            if chunk_finish:
                finish_reason = chunk_finish

        # Check for usage in final chunk (from stream_options={'include_usage': True})
        if hasattr(chunk, 'usage') and chunk.usage and user_id and operation_type:
            try:
                from src.services.token_tracking import token_tracker
                cached_tokens, cache_write_tokens = extract_cache_tokens(chunk.usage)
                # Use app context if provided (needed for generators where context may be lost)
                if app:
                    with app.app_context():
                        token_tracker.record_usage(
                            user_id=user_id,
                            operation_type=operation_type,
                            prompt_tokens=chunk.usage.prompt_tokens,
                            completion_tokens=chunk.usage.completion_tokens,
                            total_tokens=chunk.usage.total_tokens,
                            model_name=model_name or TEXT_MODEL_NAME,
                            cost=getattr(chunk.usage, 'cost', None),
                            cached_tokens=cached_tokens,
                            cache_write_tokens=cache_write_tokens
                        )
                else:
                    token_tracker.record_usage(
                        user_id=user_id,
                        operation_type=operation_type,
                        prompt_tokens=chunk.usage.prompt_tokens,
                        completion_tokens=chunk.usage.completion_tokens,
                        total_tokens=chunk.usage.total_tokens,
                        model_name=model_name or TEXT_MODEL_NAME,
                        cost=getattr(chunk.usage, 'cost', None),
                        cached_tokens=cached_tokens,
                        cache_write_tokens=cache_write_tokens
                    )
            except Exception as e:
                logger.warning(f"Failed to record streaming token usage: {e}")

        # Process content delta
        if chunk.choices and chunk.choices[0].delta.content:
            content = chunk.choices[0].delta.content
            content_buffer += content

            # Process the buffer to detect and handle thinking tags
            while True:
                if not in_thinking:
                    # Look for opening thinking tag
                    think_start = re.search(r'<think(?:ing)?>', content_buffer, re.IGNORECASE)
                    if think_start:
                        # Send any content before the thinking tag
                        before_thinking = content_buffer[:think_start.start()]
                        if before_thinking:
                            yield f"data: {json.dumps({'delta': before_thinking})}\n\n"

                        # Start capturing thinking content
                        in_thinking = True
                        content_buffer = content_buffer[think_start.end():]
                        thinking_buffer = ""
                    else:
                        # No thinking tag found, send accumulated content
                        if content_buffer:
                            yield f"data: {json.dumps({'delta': content_buffer})}\n\n"
                        content_buffer = ""
                        break
                else:
                    # We're inside a thinking tag, look for closing tag
                    think_end = re.search(r'</think(?:ing)?>', content_buffer, re.IGNORECASE)
                    if think_end:
                        # Capture thinking content up to the closing tag
                        thinking_buffer += content_buffer[:think_end.start()]

                        # Send the thinking content as a special type
                        if thinking_buffer.strip():
                            yield f"data: {json.dumps({'thinking': thinking_buffer.strip()})}\n\n"

                        # Continue processing after the closing tag
                        in_thinking = False
                        content_buffer = content_buffer[think_end.end():]
                        thinking_buffer = ""
                    else:
                        # Still inside thinking tag, accumulate content
                        thinking_buffer += content_buffer
                        content_buffer = ""
                        break

    # Handle any remaining content
    if in_thinking and thinking_buffer:
        # Unclosed thinking tag - send as thinking content
        yield f"data: {json.dumps({'thinking': thinking_buffer.strip()})}\n\n"
    elif content_buffer:
        # Regular content
        yield f"data: {json.dumps({'delta': content_buffer})}\n\n"

    # Signal the end of the stream, propagating the provider's finish_reason
    # so the frontend can flag truncated (finish_reason == 'length') responses.
    end_event = {'end_of_stream': True}
    if finish_reason:
        end_event['finish_reason'] = finish_reason
        if finish_reason == 'length':
            end_event['truncated'] = True
    yield f"data: {json.dumps(end_event)}\n\n"